public StagingTask(StagingStartMessageRequest message) { UhuruSection uhuruSection = (UhuruSection)ConfigurationManager.GetSection("uhuru"); this.TaskId = message.TaskID; this.Message = message; this.workspace = new StagingWorkspace(Path.Combine(uhuruSection.DEA.BaseDir, "staging"), message.TaskID); this.buildpacksDir = Path.GetFullPath(uhuruSection.DEA.Staging.BuildpacksDirectory); this.stagingTimeout = uhuruSection.DEA.Staging.StagingTimeoutMs; this.gitExe = Path.GetFullPath(uhuruSection.DEA.Staging.GitExecutable); var prisonInfo = new ProcessPrisonCreateInfo(); prisonInfo.Id = this.TaskId; prisonInfo.TotalPrivateMemoryLimit = (long)this.Message.Properties.Resources.MemoryMbytes * 1024 * 1024; if (uhuruSection.DEA.UseDiskQuota) { prisonInfo.DiskQuotaBytes = (long)this.Message.Properties.Resources.DiskMbytes * 1024 * 1024; prisonInfo.DiskQuotaPath = this.workspace.BaseDir; } if (uhuruSection.DEA.UploadThrottleBitsps > 0) { prisonInfo.NetworkOutboundRateLimitBitsPerSecond = uhuruSection.DEA.UploadThrottleBitsps; } this.prison = new ProcessPrison(); prison.Create(prisonInfo); }
private void StartStagingInstance(StagingInstance instance, StagingStartMessageRequest pmessage) { StagingWorkspace workspace = new StagingWorkspace(instance.Properties.Directory); try { try { instance.Lock.EnterWriteLock(); instance.Properties.UseDiskQuota = this.useDiskQuota; instance.Properties.UploadThrottleBitsps = this.uploadThrottleBitsps; UriBuilder streamingLog = new UriBuilder(); streamingLog.Host = this.ExternalHost; streamingLog.Scheme = "http"; streamingLog.Path = string.Format("/staging_tasks/{0}/file_path", pmessage.TaskID); streamingLog.Query = string.Format("path={0}×tamp={1}", workspace.StagingLogSuffix, RubyCompatibility.DateTimeToEpochSeconds(DateTime.Now)); instance.Properties.StreamingLogUrl = DEAUtilities.GetHmacedUri(streamingLog.Uri.ToString(), this.directoryServerHmacKey, new string[] { "path", "timestamp" }).ToString(); instance.Workspace = workspace; instance.Properties.TaskLog = workspace.StagingLogPath; } finally { instance.Lock.ExitWriteLock(); } instance.AfterSetup += new StagingInstance.StagingTaskEventHandler(this.AfterStagingSetup); Logger.Info("Started staging task {0}", instance.Properties.TaskId); try { instance.SetupStagingEnvironment(); } catch (Exception ex) { Logger.Error("Error setting up staging environment: ", ex.ToString()); throw ex; } instance.UnpackDroplet(); instance.PrepareStagingDirs(); instance.GetBuildpack(pmessage, this.gitPath, this.buildpacksDir); this.stagingTaskRegistry.ScheduleSnapshotStagingState(); try { Logger.Info("Staging task {0}: Running compilation script", pmessage.TaskID); instance.CreatePrison(); this.stagingTaskRegistry.ScheduleSnapshotStagingState(); instance.CompileProcess = instance.Buildpack.StartCompile(instance.Prison); instance.Lock.EnterWriteLock(); instance.Properties.Start = DateTime.Now; } finally { if(instance.Lock.IsWriteLockHeld) { instance.Lock.ExitWriteLock(); } } } catch (Exception ex) { instance.StagingException = ex; instance.Properties.Stopped = true; Logger.Error(ex.ToString()); } }
private void StagingStartHandler(string message, string reply, string subject) { StagingStartMessageRequest pmessage; StagingInstance instance; try { this.stagingTaskRegistry.Lock.EnterWriteLock(); if (this.shuttingDown) { return; } Logger.Debug("DEA Received staging message: {0}", message); pmessage = new StagingStartMessageRequest(); try { pmessage.FromJsonIntermediateObject(JsonConvertibleObject.DeserializeFromJson(message)); } catch (Exception e) { Logger.Error("Ignoring staging.start request. Unable to parse message. Exception: {0}", e.ToString()); return; } long memoryMbytes = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.MemoryMbytes != null ? pmessage.Properties.Resources.MemoryMbytes.Value : Monitoring.DefaultAppMemoryMbytes; long diskMbytes = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.DiskMbytes != null ? pmessage.Properties.Resources.DiskMbytes.Value : Monitoring.DefaultAppDiskMbytes; long fds = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.FileDescriptors != null ? pmessage.Properties.Resources.FileDescriptors.Value : Monitoring.DefaultAppFds; if (this.monitoring.MemoryReservedMbytes + memoryMbytes > this.monitoring.MaxMemoryMbytes || this.monitoring.Clients >= this.monitoring.MaxClients) { Logger.Info(Strings.Donothaveroomforthisclient); return; } instance = this.stagingTaskRegistry.CreateStagingInstance(pmessage); instance.Properties.MemoryQuotaBytes = memoryMbytes * 1024 * 1024; instance.Properties.DiskQuotaBytes = diskMbytes * 1024 * 1024; instance.Properties.FDSQuota = fds; instance.Properties.Directory = Path.Combine(this.fileResources.StagingDir, pmessage.TaskID); instance.Properties.TaskId = pmessage.TaskID; instance.Properties.Reply = reply; this.monitoring.AddInstanceResources(instance); } finally { this.stagingTaskRegistry.Lock.ExitWriteLock(); } ThreadPool.QueueUserWorkItem(delegate(object data) { this.StartStagingInstance(instance, pmessage); }); }
public StagingInstance CreateStagingInstance(StagingStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } StagingInstance instance = new StagingInstance(); string instanceId = Credentials.GenerateSecureGuid().ToString("N"); string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N"); instance.StartMessage = message.StartMessage; instance.Properties.InstanceId = instanceId; instance.Properties.TaskId = message.TaskID; instance.Properties.AppId = message.AppID; instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI; instance.Properties.BuildpackCacheUploadURI = message.BuildpackCacheUploadURI; instance.Properties.DownloadURI = message.DownloadURI; instance.Properties.UploadURI = message.UploadURI; if (message.Properties.Meta != null) { if (message.Properties.Meta.Command != null) { instance.Properties.MetaCommand = message.Properties.Meta.Command; } } this.AddStagingInstance(instance); return instance; }
public void GetBuildpack(StagingStartMessageRequest message, string gitPath, string buildpacksDir) { try { this.CreatePrison(); if (message.Properties.Buildpack != null) { Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.Properties.TaskId, message.Properties.Buildpack); Directory.CreateDirectory(Path.Combine(this.Workspace.TempDir, "buildpacks")); string buildpackPath = Path.Combine(this.Workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(message.Properties.Buildpack).LocalPath)); string command = string.Format("\"{0}\" clone --quiet --recursive {1} {2}", gitPath, message.Properties.Buildpack, buildpackPath); Logger.Debug(command); int success = Command.ExecuteCommand(command, this.Workspace.TempDir); if (success != 0) { throw new Exception(string.Format("Failed to git clone buildpack. Exit code: {0}", success)); } this.Buildpack = new Buildpack(buildpackPath, Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool detected = this.Buildpack.Detect(this.Prison); if (!detected) { throw new Exception("Buildpack does not support this application."); } } else { Logger.Info("Staging task {0}: Detecting buildpack", this.Properties.TaskId); foreach (string dir in Directory.EnumerateDirectories(buildpacksDir)) { DEAUtilities.DirectoryCopy(dir, Path.Combine(this.Workspace.TempDir, "buildpack"), true); Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool success = bp.Detect(this.Prison); if (success) { this.Buildpack = bp; break; } else { Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true); } } if (this.Buildpack == null) { throw new Exception("Unable to detect a supported application type"); } Logger.Info("Staging task {0}: Detected buildpack {1}", this.Properties.TaskId, this.Buildpack.Name); } this.Properties.DetectedBuildpack = this.Buildpack.Name; } finally { if (this.Prison.Created) { this.Prison.Destroy(); } } }