public void GetBuildpack(StagingStartMessageRequest message, string gitPath, string buildpacksDir) { if (message.Properties.Buildpack != null) { Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.Properties.TaskId, message.Properties.Buildpack); Directory.CreateDirectory(Path.Combine(this.Workspace.TempDir, "buildpacks")); string buildpackPath = Path.Combine(this.Workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(message.Properties.Buildpack).LocalPath)); string command = string.Format("\"{0}\" clone --quiet --recursive {1} {2}", gitPath, message.Properties.Buildpack, buildpackPath); Logger.Debug(command); int success = Command.ExecuteCommand(command, this.Workspace.TempDir); if (success != 0) { throw new Exception(string.Format("Failed to git clone buildpack. Exit code: {0}", success)); } this.Buildpack = new Buildpack(buildpackPath, Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool detected = this.Buildpack.Detect(this.Container); if (!detected) { throw new Exception("Buildpack does not support this application."); } } else { Logger.Info("Staging task {0}: Detecting buildpack", this.Properties.TaskId); foreach (string dir in Directory.EnumerateDirectories(buildpacksDir)) { DEAUtilities.DirectoryCopy(dir, Path.Combine(this.Workspace.TempDir, "buildpack"), true); Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool success = bp.Detect(this.Container); if (success) { this.Buildpack = bp; break; } else { Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true); } } if (this.Buildpack == null) { throw new Exception("Unable to detect a supported application type"); } Logger.Info("Staging task {0}: Detected buildpack {1}", this.Properties.TaskId, this.Buildpack.Name); } this.Properties.DetectedBuildpack = this.Buildpack.Name; }
public void GetBuildpack(StagingStartMessageRequest message, string gitPath, string buildpacksDir, string adminBuildpackDir) { string buildpackUrl = message.Properties.Buildpack ?? message.Properties.BuildpackGitUrl; if (buildpackUrl != null) { Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.Properties.TaskId, message.Properties.Buildpack); Directory.CreateDirectory(Path.Combine(this.Workspace.TempDir, "buildpacks")); string buildpackPath = Path.Combine(this.Workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(message.Properties.Buildpack).LocalPath)); string command = string.Format("set GIT_TERMINAL_PROMPT=0& \"{0}\" clone --quiet --recursive {1} {2}", gitPath, message.Properties.Buildpack, buildpackPath); Logger.Debug(command); int success = Command.ExecuteCommand(command, this.Workspace.TempDir); if (success != 0) { throw new Exception(string.Format("Failed to git clone buildpack. Exit code: {0}", success)); } this.Buildpack = new Buildpack(buildpackPath, Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool detected = this.Buildpack.Detect(this.Container); if (!detected) { throw new Exception("Buildpack does not support this application."); } } else if (message.Properties.BuildpackKey != null) { Logger.Info("Staging task {0}: Using admin buildpack {1}", this.Properties.TaskId, message.Properties.BuildpackKey); TryCleanupUnusedAdminBuildpacks(message.AdminBuildpacks, adminBuildpackDir); var adminBuildpack = message.AdminBuildpacks.First(i => i.Key == message.Properties.BuildpackKey); InitializeAdminBuildpack(adminBuildpack, adminBuildpackDir, false); this.Buildpack = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool detected = this.Buildpack.Detect(this.Container); if (!detected) { throw new InvalidOperationException("Buildpack does not support this application."); } } else { Logger.Info("Staging task {0}: Detecting buildpack", this.Properties.TaskId); if (message.AdminBuildpacks != null) { TryCleanupUnusedAdminBuildpacks(message.AdminBuildpacks, adminBuildpackDir); foreach (var adminBuildpack in message.AdminBuildpacks) { if (!InitializeAdminBuildpack(adminBuildpack, adminBuildpackDir, true)) { continue; } Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool success = bp.Detect(this.Container); if (success) { this.Buildpack = bp; break; } else { Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true); } } } if (this.Buildpack == null) { if (Directory.Exists(buildpacksDir)) { List<string> systemBuildpacks = Directory.EnumerateDirectories(buildpacksDir).ToList(); foreach (string dir in systemBuildpacks) { DEAUtilities.DirectoryCopy(dir, Path.Combine(this.Workspace.TempDir, "buildpack"), true); Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath); bool success = bp.Detect(this.Container); if (success) { this.Buildpack = bp; break; } else { Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true); } } } } if (this.Buildpack == null) { throw new InvalidOperationException("Unable to detect a supported application type."); } Logger.Info("Staging task {0}: Detected buildpack {1}", this.Properties.TaskId, this.Buildpack.Name); } this.Properties.DetectedBuildpack = this.Buildpack.Name; }
private void StagingStartHandler(string message, string reply, string subject) { StagingStartMessageRequest pmessage; StagingInstance instance; try { this.stagingTaskRegistry.Lock.EnterWriteLock(); if (this.shuttingDown) { return; } Logger.Debug("DEA Received staging message: {0}", message); pmessage = new StagingStartMessageRequest(); try { pmessage.FromJsonIntermediateObject(JsonConvertibleObject.DeserializeFromJson(message)); } catch (Exception e) { Logger.Error("Ignoring staging.start request. Unable to parse message. Exception: {0}", e.ToString()); return; } long memoryMbytes = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.MemoryMbytes != null ? pmessage.Properties.Resources.MemoryMbytes.Value : Monitoring.DefaultAppMemoryMbytes; long diskMbytes = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.DiskMbytes != null ? pmessage.Properties.Resources.DiskMbytes.Value : Monitoring.DefaultAppDiskMbytes; long fds = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.FileDescriptors != null ? pmessage.Properties.Resources.FileDescriptors.Value : Monitoring.DefaultAppFds; if (this.monitoring.MemoryReservedMbytes + memoryMbytes > this.monitoring.MaxMemoryMbytes || this.monitoring.Clients >= this.monitoring.MaxClients) { Logger.Info(Strings.Donothaveroomforthisclient); return; } instance = this.stagingTaskRegistry.CreateStagingInstance(pmessage); instance.Properties.MemoryQuotaBytes = memoryMbytes * 1024 * 1024; instance.Properties.DiskQuotaBytes = diskMbytes * 1024 * 1024; instance.Properties.FDSQuota = fds; instance.Properties.Directory = Path.Combine(this.fileResources.StagingDir, pmessage.TaskID); instance.Properties.TaskId = pmessage.TaskID; instance.Properties.Reply = reply; instance.Properties.InitializedTime = DateTime.Now; instance.StopRequested = false; this.monitoring.AddInstanceResources(instance); } finally { this.stagingTaskRegistry.Lock.ExitWriteLock(); } ThreadPool.QueueUserWorkItem(delegate(object data) { this.StartStagingInstance(instance, pmessage); }); }
private void StartStagingInstance(StagingInstance instance, StagingStartMessageRequest pmessage) { StagingWorkspace workspace = new StagingWorkspace(instance.Properties.Directory); try { try { instance.Lock.EnterWriteLock(); instance.Properties.UseDiskQuota = this.useDiskQuota; instance.Properties.UploadThrottleBitsps = this.uploadThrottleBitsps; UriBuilder streamingLog = new UriBuilder(); streamingLog.Host = this.ExternalHost; streamingLog.Scheme = "http"; streamingLog.Path = string.Format("/staging_tasks/{0}/file_path", pmessage.TaskID); streamingLog.Query = string.Format("path={0}×tamp={1}", workspace.StagingLogSuffix, RubyCompatibility.DateTimeToEpochSeconds(DateTime.Now)); instance.Properties.StreamingLogUrl = DEAUtilities.GetHmacedUri(streamingLog.Uri.ToString(), this.directoryServerHmacKey, new string[] { "path", "timestamp" }).ToString(); instance.Workspace = workspace; instance.Properties.TaskLog = workspace.StagingLogPath; } finally { instance.Lock.ExitWriteLock(); } instance.AfterSetup += new StagingInstance.StagingTaskEventHandler(this.AfterStagingSetup); Logger.Info("Started staging task {0}", instance.Properties.TaskId); try { instance.SetupStagingEnvironment(); } catch (Exception ex) { Logger.Error("Error setting up staging environment: ", ex.ToString()); throw ex; } instance.UnpackDroplet(); instance.PrepareStagingDirs(); if (File.Exists(this.logyardUidPath)) { LogyardInstanceRequest logyardMsg = new LogyardInstanceRequest(); logyardMsg.AppGUID = pmessage.AppID; logyardMsg.AppName = pmessage.StartMessage.Name; logyardMsg.AppSpace = ""; logyardMsg.DockerId = instance.Properties.InstanceId; logyardMsg.Index = -1; logyardMsg.LogFiles = new Dictionary<string, string>() { { "staging", Path.Combine("staging", instance.Workspace.StagingLogSuffix) } }; logyardMsg.Type = "staging"; logyardMsg.RootPath = instance.Workspace.BaseDir; string logyardId = File.ReadAllText(this.logyardUidPath).Trim(); this.deaReactor.SendLogyardNotification(logyardId, logyardMsg.SerializeToJson()); } instance.CreatePrison(); if (pmessage.Properties.Environment != null) { Dictionary<string, string> stagingEnvVars = this.ParseEnvironmnetVariables(pmessage.Properties.Environment); instance.Container.User.SetUserEnvironmentVariables(stagingEnvVars); } instance.GetBuildpack(pmessage, this.gitPath, this.buildpacksDir); this.stagingTaskRegistry.ScheduleSnapshotStagingState(); try { Logger.Info("Staging task {0}: Running compilation script", pmessage.TaskID); this.stagingTaskRegistry.ScheduleSnapshotStagingState(); instance.CompileProcess = instance.Buildpack.StartCompile(instance.Container); instance.Lock.EnterWriteLock(); instance.Properties.Start = DateTime.Now; } finally { if (instance.Lock.IsWriteLockHeld) { instance.Lock.ExitWriteLock(); } } } catch (Exception ex) { instance.StagingException = ex; instance.Properties.Stopped = true; Logger.Error(ex.ToString()); } }
public StagingInstance CreateStagingInstance(StagingStartMessageRequest message) { if (message == null) { throw new ArgumentNullException("message"); } StagingInstance instance = new StagingInstance(); string instanceId = Credentials.GenerateSecureGuid().ToString("N"); string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N"); instance.StartMessage = message.StartMessage; instance.Properties.InstanceId = instanceId; instance.Properties.TaskId = message.TaskID; instance.Properties.AppId = message.AppID; instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI; instance.Properties.BuildpackCacheUploadURI = message.BuildpackCacheUploadURI; instance.Properties.DownloadURI = message.DownloadURI; instance.Properties.UploadURI = message.UploadURI; if (message.Properties.Meta != null) { if (message.Properties.Meta.Command != null) { instance.Properties.MetaCommand = message.Properties.Meta.Command; } } this.AddStagingInstance(instance); return instance; }