Exemplo n.º 1
0
        /// <summary>
        /// Untracks the memory used by the instance and flags it/
        /// </summary>
        /// <param name="instance">The instance to be untracked.</param>
        public void RemoveInstanceResources(StagingInstance instance)
        {
            if (instance == null)
            {
                throw new ArgumentNullException("instance");
            }

            try
            {
                this.Lock.EnterWriteLock();
                instance.Lock.EnterWriteLock();

                if (instance.Properties.ResourcesTracked)
                {
                    instance.Properties.ResourcesTracked = false;
                    this.MemoryReservedMbytes           -= instance.Properties.MemoryQuotaBytes / 1024 / 1024;
                    this.Clients--;
                }
            }
            finally
            {
                instance.Lock.ExitWriteLock();
                this.Lock.ExitWriteLock();
            }
        }
Exemplo n.º 2
0
        public StagingInstance CreateStagingInstance(StagingStartMessageRequest message)
        {
            if (message == null)
            {
                throw new ArgumentNullException("message");
            }

            StagingInstance instance = new StagingInstance();

            string instanceId        = Credentials.GenerateSecureGuid().ToString("N");
            string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");

            instance.StartMessage = message.StartMessage;

            instance.Properties.InstanceId = instanceId;
            instance.Properties.TaskId     = message.TaskID;
            instance.Properties.AppId      = message.AppID;
            instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI;
            instance.Properties.BuildpackCacheUploadURI   = message.BuildpackCacheUploadURI;
            instance.Properties.DownloadURI = message.DownloadURI;
            instance.Properties.UploadURI   = message.UploadURI;

            if (message.Properties.Meta != null)
            {
                if (message.Properties.Meta.Command != null)
                {
                    instance.Properties.MetaCommand = message.Properties.Meta.Command;
                }
            }

            this.AddStagingInstance(instance);

            return(instance);
        }
Exemplo n.º 3
0
        public void AddStagingInstance(StagingInstance instance)
        {
            if (instance == null)
            {
                throw new ArgumentNullException("instance");
            }

            try
            {
                this.Lock.EnterWriteLock();
                instance.Lock.EnterReadLock();

                if (!this.Tasks.ContainsKey(instance.Properties.TaskId))
                {
                    this.Tasks.Add(instance.Properties.TaskId, instance);
                }
            }
            finally
            {
                instance.Lock.ExitReadLock();
                this.Lock.ExitWriteLock();
            }

            this.ScheduleSnapshotStagingState();
        }
Exemplo n.º 4
0
        private void CleanupStagingInstances()
        {
            if (!File.Exists(this.stagingTaskRegistry.StagingStateFile))
            {
                return;
            }

            object[] instances = JsonConvertibleObject.DeserializeFromJsonArray(File.ReadAllText(this.stagingTaskRegistry.StagingStateFile));

            foreach (object obj in instances)
            {
                StagingInstance instance = null;
                try
                {
                    instance = new StagingInstance();
                    instance.Properties.FromJsonIntermediateObject(obj);

                    var prisonInfo = new ProcessPrisonCreateInfo();
                    prisonInfo.Id = instance.Properties.InstanceId;

                    prisonInfo.TotalPrivateMemoryLimitBytes = instance.Properties.MemoryQuotaBytes;
                    prisonInfo.WindowsPassword = instance.Properties.WindowsPassword;

                    if (this.useDiskQuota)
                    {
                        prisonInfo.DiskQuotaBytes = instance.Properties.DiskQuotaBytes;
                        prisonInfo.DiskQuotaPath = instance.Properties.Directory;
                    }

                    Logger.Info("Recovering Process Prisson: {0}", prisonInfo.Id);

                    instance.Prison.Attach(prisonInfo);
                    foreach (Process p in instance.Prison.jobObject.GetJobProcesses())
                    {
                        if (!p.HasExited)
                        {
                            p.Kill();
                        }
                    }
                    if (instance.Prison.Created)
                    {
                        instance.Prison.Destroy();
                    }
                    instance.Workspace = new StagingWorkspace(instance.Properties.Directory);
                    instance.Cleanup();
                }
                catch (Exception ex)
                {
                    Logger.Warning("Error deleting staging environment for task {0}: {1}", instance.Properties.TaskId, ex.ToString());
                }
                finally
                {
                    this.stagingTaskRegistry.RemoveStagingInstance(instance);
                    if (instance != null)
                    {
                        instance.Dispose();
                    }
                }
            }
        }
Exemplo n.º 5
0
 private void AfterStagingSetup(StagingInstance instance)
 {
     StagingStartMessageResponse response = new StagingStartMessageResponse();
     response.TaskId = instance.Properties.TaskId;
     response.TaskStreamingLogURL = instance.Properties.StreamingLogUrl;
     if (instance.StagingException != null)
     {
         response.Error = instance.StagingException.ToString();
     }
     this.deaReactor.SendReply(instance.Properties.Reply, response.SerializeToJson());
     Logger.Debug("Staging task {0}: sent reply {1}", instance.Properties.TaskId, response.SerializeToJson());
 }
Exemplo n.º 6
0
        private void AfterStagingFinished(StagingInstance instance)
        {
            StagingStartMessageResponse response = new StagingStartMessageResponse();
            try
            {
                if (instance.StagingException == null)
                {
                    try
                    {
                        Logger.Info("Staging task {0}: Saving buildpackInfo", instance.Properties.TaskId);
                        StagingInfo.SaveBuildpackInfo(Path.Combine(instance.Workspace.StagedDir, StagingWorkspace.StagingInfo), instance.Buildpack.Name, instance.GetStartCommand());
                        this.stagingTaskRegistry.ScheduleSnapshotStagingState();

                        Logger.Debug("Staging task {0}: Packing droplet {1}", instance.Properties.TaskId, instance.Workspace.StagedDropletPath);
                        Directory.CreateDirectory(instance.Workspace.StagedDropletDir);
                        string tempFile = Path.ChangeExtension(instance.Workspace.StagedDropletPath, "tar");
                        DEAUtilities.TarDirectory(instance.Workspace.StagedDir, tempFile);
                        DEAUtilities.GzipFile(tempFile, instance.Workspace.StagedDropletPath);
                        File.Delete(tempFile);

                        if (File.Exists(instance.Workspace.StagedDropletPath))
                        {
                            using (Stream stream = File.OpenRead(instance.Workspace.StagedDropletPath))
                            {
                                using (SHA1 sha = SHA1.Create())
                                {
                                    response.DropletSHA = BitConverter.ToString(sha.ComputeHash(stream)).Replace("-", string.Empty);
                                }
                            }
                        }

                        this.StartStagedDropletInstance(instance, response.DropletSHA);

                        Uri uri = new Uri(instance.Properties.UploadURI);
                        Logger.Debug("Staging task {0}: Uploading droplet {1} to {2}", instance.Properties.TaskId, instance.Workspace.StagedDropletPath, instance.Properties.UploadURI);
                        DEAUtilities.HttpUploadFile(instance.Properties.UploadURI, new FileInfo(instance.Workspace.StagedDropletPath), "upload[droplet]", "application/octet-stream", uri.UserInfo);
                    }
                    catch (Exception ex)
                    {
                        instance.StagingException = ex;
                    }
                    try
                    {
                        Directory.CreateDirectory(instance.Workspace.Cache);
                        string tempFile = Path.ChangeExtension(instance.Workspace.StagedBuildpackCachePath, "tar");
                        DEAUtilities.TarDirectory(instance.Workspace.Cache, tempFile);
                        DEAUtilities.GzipFile(tempFile, instance.Workspace.StagedBuildpackCachePath);
                        Uri uri = new Uri(instance.Properties.BuildpackCacheUploadURI);
                        Logger.Debug("Staging task {0}: Uploading buildpack cache {1} to {2}", instance.Properties.TaskId, instance.Workspace.StagedBuildpackCachePath, instance.Properties.BuildpackCacheUploadURI);
                        DEAUtilities.HttpUploadFile(instance.Properties.BuildpackCacheUploadURI, new FileInfo(instance.Workspace.StagedBuildpackCachePath), "upload[droplet]", "application/octet-stream", uri.UserInfo);
                    }
                    catch
                    {
                        Logger.Debug("Staging task {0}: Cannot pack buildpack cache", instance.Properties.TaskId);
                    }
                }

                if (instance.StagingException != null)
                {
                    response.Error = instance.StagingException.ToString();
                }

                response.TaskId = instance.Properties.TaskId;

                // try to read log. don't throw exception if it fails
                try
                {
                    response.TaskLog = File.ReadAllText(instance.Properties.TaskLog);
                }
                catch { }

                if (instance.Properties.DetectedBuildpack != null)
                {
                    response.DetectedBuildpack = instance.Properties.DetectedBuildpack;
                }

                this.deaReactor.SendReply(instance.Properties.Reply, response.SerializeToJson());
                Logger.Debug("Staging task {0}: sent reply {1}", instance.Properties.TaskId, response.SerializeToJson());
            }
            finally
            {
                Logger.Debug("Cleaning up directory {0}", instance.Workspace.BaseDir);
                instance.Cleanup();
            }
        }
Exemplo n.º 7
0
 private void StopStaging(StagingInstance instance, string reply_to)
 {
     try
     {
         if (instance.Properties.Stopped)
         {
             return;
         }
         instance.Lock.EnterWriteLock();
         instance.Properties.Stopped = true;
         StagingStartMessageResponse response = new StagingStartMessageResponse();
         response.TaskId = instance.Properties.TaskId;
         this.deaReactor.SendReply(reply_to, response.SerializeToJson());
     }
     catch (Exception ex)
     {
         Logger.Error("Could not stop staging task {0}: {1}", instance.Properties.TaskId, ex.ToString());
     }
     finally
     {
         instance.Lock.ExitWriteLock();
     }
 }
Exemplo n.º 8
0
        private void StartStagingInstance(StagingInstance instance, StagingStartMessageRequest pmessage)
        {
            StagingWorkspace workspace = new StagingWorkspace(instance.Properties.Directory);
            try
            {
                try
                {
                    instance.Lock.EnterWriteLock();

                    instance.Properties.UseDiskQuota = this.useDiskQuota;
                    instance.Properties.UploadThrottleBitsps = this.uploadThrottleBitsps;

                    UriBuilder streamingLog = new UriBuilder();
                    streamingLog.Host = this.ExternalHost;
                    streamingLog.Scheme = "http";
                    streamingLog.Path = string.Format("/staging_tasks/{0}/file_path", pmessage.TaskID);
                    streamingLog.Query = string.Format("path={0}&timestamp={1}", workspace.StagingLogSuffix, RubyCompatibility.DateTimeToEpochSeconds(DateTime.Now));

                    instance.Properties.StreamingLogUrl = DEAUtilities.GetHmacedUri(streamingLog.Uri.ToString(), this.directoryServerHmacKey, new string[] { "path", "timestamp" }).ToString();
                    instance.Workspace = workspace;
                    instance.Properties.TaskLog = workspace.StagingLogPath;
                }
                finally
                {
                    instance.Lock.ExitWriteLock();
                }

                instance.AfterSetup += new StagingInstance.StagingTaskEventHandler(this.AfterStagingSetup);

                Logger.Info("Started staging task {0}", instance.Properties.TaskId);
                try
                {
                    instance.SetupStagingEnvironment();
                }
                catch (Exception ex)
                {
                    Logger.Error("Error setting up staging environment: ", ex.ToString());
                    throw ex;
                }

                instance.UnpackDroplet();
                instance.PrepareStagingDirs();

                instance.GetBuildpack(pmessage, this.gitPath, this.buildpacksDir);
                this.stagingTaskRegistry.ScheduleSnapshotStagingState();

                try
                {
                    Logger.Info("Staging task {0}: Running compilation script", pmessage.TaskID);

                    instance.CreatePrison();
                    this.stagingTaskRegistry.ScheduleSnapshotStagingState();
                    instance.CompileProcess = instance.Buildpack.StartCompile(instance.Prison);

                    instance.Lock.EnterWriteLock();
                    instance.Properties.Start = DateTime.Now;
                }
                finally
                {
                    if(instance.Lock.IsWriteLockHeld)
                    {
                        instance.Lock.ExitWriteLock();
                    }
                }
            }
            catch (Exception ex)
            {
                instance.StagingException = ex;
                instance.Properties.Stopped = true;
                Logger.Error(ex.ToString());
            }
        }
Exemplo n.º 9
0
        private void StartStagedDropletInstance(StagingInstance stagingInstance, string dropletSha)
        {
            DropletInstance instance;

            try
            {
                this.droplets.Lock.EnterWriteLock();

                if (this.shuttingDown)
                {
                    return;
                }

                string tgzFile = Path.Combine(this.fileResources.StagedDir, dropletSha + ".tgz");
                Logger.Info("Copying droplet to {0}", tgzFile);
                File.Copy(stagingInstance.Workspace.StagedDropletPath, tgzFile);

                long memoryMbytes = stagingInstance.StartMessage.Limits != null && stagingInstance.StartMessage.Limits.MemoryMbytes != null ? stagingInstance.StartMessage.Limits.MemoryMbytes.Value : Monitoring.DefaultAppMemoryMbytes;
                long diskMbytes = stagingInstance.StartMessage.Limits != null && stagingInstance.StartMessage.Limits.DiskMbytes != null ? stagingInstance.StartMessage.Limits.DiskMbytes.Value : Monitoring.DefaultAppDiskMbytes;
                long fds = stagingInstance.StartMessage.Limits != null && stagingInstance.StartMessage.Limits.FileDescriptors != null ? stagingInstance.StartMessage.Limits.FileDescriptors.Value : Monitoring.DefaultAppFds;

                if (this.monitoring.MemoryReservedMbytes + memoryMbytes > this.monitoring.MaxMemoryMbytes || this.monitoring.Clients >= this.monitoring.MaxClients)
                {
                    Logger.Info(Strings.Donothaveroomforthisclient);
                    return;
                }

                instance = this.droplets.CreateDropletInstance(stagingInstance.StartMessage);
                instance.Properties.MemoryQuotaBytes = memoryMbytes * 1024 * 1024;
                instance.Properties.DiskQuotaBytes = diskMbytes * 1024 * 1024;
                instance.Properties.FDSQuota = fds;

                instance.Properties.Staged = instance.Properties.Name + "-" + instance.Properties.InstanceIndex + "-" + instance.Properties.InstanceId;
                instance.Properties.Directory = Path.Combine(this.fileResources.AppsDir, instance.Properties.Staged);

                if (!string.IsNullOrEmpty(instance.Properties.DebugMode))
                {
                    instance.Properties.DebugPort = NetworkInterface.GrabEphemeralPort();
                    instance.Properties.DebugIP = Host;
                }

                instance.Properties.Port = NetworkInterface.GrabEphemeralPort();
                instance.Properties.EnvironmentVariables = this.SetupInstanceEnv(instance, stagingInstance.StartMessage.Environment, stagingInstance.StartMessage.Services);

                this.monitoring.AddInstanceResources(instance);
            }
            finally
            {
                this.droplets.Lock.ExitWriteLock();
            }

            ThreadPool.QueueUserWorkItem(delegate(object data)
            {
                this.StartDropletInstance(instance, dropletSha, stagingInstance.StartMessage.ExecutableFile, stagingInstance.StartMessage.ExecutableUri);
            });
        }
Exemplo n.º 10
0
        public void AddStagingInstance(StagingInstance instance)
        {
            if (instance == null)
            {
                throw new ArgumentNullException("instance");
            }

            try
            {
                this.Lock.EnterWriteLock();
                instance.Lock.EnterReadLock();

                if (!this.Tasks.ContainsKey(instance.Properties.TaskId))
                {
                    this.Tasks.Add(instance.Properties.TaskId, instance);
                }
            }
            finally
            {
                instance.Lock.ExitReadLock();
                this.Lock.ExitWriteLock();
            }

            this.ScheduleSnapshotStagingState();
        }
Exemplo n.º 11
0
        public StagingInstance CreateStagingInstance(StagingStartMessageRequest message)
        {
            if (message == null)
            {
                throw new ArgumentNullException("message");
            }

            StagingInstance instance = new StagingInstance();

            string instanceId = Credentials.GenerateSecureGuid().ToString("N");
            string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");

            instance.StartMessage = message.StartMessage;

            instance.Properties.InstanceId = instanceId;
            instance.Properties.TaskId = message.TaskID;
            instance.Properties.AppId = message.AppID;
            instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI;
            instance.Properties.BuildpackCacheUploadURI = message.BuildpackCacheUploadURI;
            instance.Properties.DownloadURI = message.DownloadURI;
            instance.Properties.UploadURI = message.UploadURI;

            if (message.Properties.Meta != null)
            {
                if (message.Properties.Meta.Command != null)
                {
                    instance.Properties.MetaCommand = message.Properties.Meta.Command;
                }
            }

            this.AddStagingInstance(instance);

            return instance;
        }
Exemplo n.º 12
0
        /// <summary>
        /// Untracks the memory used by the instance and flags it/
        /// </summary>
        /// <param name="instance">The instance to be untracked.</param>
        public void RemoveInstanceResources(StagingInstance instance)
        {
            if (instance == null)
            {
                throw new ArgumentNullException("instance");
            }

            try
            {
                this.Lock.EnterWriteLock();
                instance.Lock.EnterWriteLock();

                if (instance.Properties.ResourcesTracked)
                {
                    instance.Properties.ResourcesTracked = false;
                    this.MemoryReservedMbytes -= instance.Properties.MemoryQuotaBytes / 1024 / 1024;
                    this.Clients--;
                }
            }
            finally
            {
                instance.Lock.ExitWriteLock();
                this.Lock.ExitWriteLock();
            }
        }