public void GetBuildpack(StagingStartMessageRequest message, string gitPath, string buildpacksDir)
{
if (message.Properties.Buildpack != null)
{
Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.Properties.TaskId, message.Properties.Buildpack);
Directory.CreateDirectory(Path.Combine(this.Workspace.TempDir, "buildpacks"));
string buildpackPath = Path.Combine(this.Workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(message.Properties.Buildpack).LocalPath));
string command = string.Format("\"{0}\" clone --quiet --recursive {1} {2}", gitPath, message.Properties.Buildpack, buildpackPath);
Logger.Debug(command);
int success = Command.ExecuteCommand(command, this.Workspace.TempDir);
if (success != 0)
{
throw new Exception(string.Format("Failed to git clone buildpack. Exit code: {0}", success));
}
this.Buildpack = new Buildpack(buildpackPath, Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool detected = this.Buildpack.Detect(this.Container);
if (!detected)
{
throw new Exception("Buildpack does not support this application.");
}
}
else
{
Logger.Info("Staging task {0}: Detecting buildpack", this.Properties.TaskId);
foreach (string dir in Directory.EnumerateDirectories(buildpacksDir))
{
DEAUtilities.DirectoryCopy(dir, Path.Combine(this.Workspace.TempDir, "buildpack"), true);
Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool success = bp.Detect(this.Container);
if (success)
{
this.Buildpack = bp;
break;
}
else
{
Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true);
}
}
if (this.Buildpack == null)
{
throw new Exception("Unable to detect a supported application type");
}
Logger.Info("Staging task {0}: Detected buildpack {1}", this.Properties.TaskId, this.Buildpack.Name);
}
this.Properties.DetectedBuildpack = this.Buildpack.Name;
}
public void GetBuildpack(StagingStartMessageRequest message, string gitPath, string buildpacksDir, string adminBuildpackDir)
{
string buildpackUrl = message.Properties.Buildpack ?? message.Properties.BuildpackGitUrl;
if (buildpackUrl != null)
{
Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.Properties.TaskId, message.Properties.Buildpack);
Directory.CreateDirectory(Path.Combine(this.Workspace.TempDir, "buildpacks"));
string buildpackPath = Path.Combine(this.Workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(message.Properties.Buildpack).LocalPath));
string command = string.Format("set GIT_TERMINAL_PROMPT=0& \"{0}\" clone --quiet --recursive {1} {2}", gitPath, message.Properties.Buildpack, buildpackPath);
Logger.Debug(command);
int success = Command.ExecuteCommand(command, this.Workspace.TempDir);
if (success != 0)
{
throw new Exception(string.Format("Failed to git clone buildpack. Exit code: {0}", success));
}
this.Buildpack = new Buildpack(buildpackPath, Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool detected = this.Buildpack.Detect(this.Container);
if (!detected)
{
throw new Exception("Buildpack does not support this application.");
}
}
else if (message.Properties.BuildpackKey != null)
{
Logger.Info("Staging task {0}: Using admin buildpack {1}", this.Properties.TaskId, message.Properties.BuildpackKey);
TryCleanupUnusedAdminBuildpacks(message.AdminBuildpacks, adminBuildpackDir);
var adminBuildpack = message.AdminBuildpacks.First(i => i.Key == message.Properties.BuildpackKey);
InitializeAdminBuildpack(adminBuildpack, adminBuildpackDir, false);
this.Buildpack = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool detected = this.Buildpack.Detect(this.Container);
if (!detected)
{
throw new InvalidOperationException("Buildpack does not support this application.");
}
}
else
{
Logger.Info("Staging task {0}: Detecting buildpack", this.Properties.TaskId);
if (message.AdminBuildpacks != null)
{
TryCleanupUnusedAdminBuildpacks(message.AdminBuildpacks, adminBuildpackDir);
foreach (var adminBuildpack in message.AdminBuildpacks)
{
if (!InitializeAdminBuildpack(adminBuildpack, adminBuildpackDir, true))
{
continue;
}
Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool success = bp.Detect(this.Container);
if (success)
{
this.Buildpack = bp;
break;
}
else
{
Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true);
}
}
}
if (this.Buildpack == null)
{
if (Directory.Exists(buildpacksDir))
{
List<string> systemBuildpacks = Directory.EnumerateDirectories(buildpacksDir).ToList();
foreach (string dir in systemBuildpacks)
{
DEAUtilities.DirectoryCopy(dir, Path.Combine(this.Workspace.TempDir, "buildpack"), true);
Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool success = bp.Detect(this.Container);
if (success)
{
this.Buildpack = bp;
break;
}
else
{
Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true);
}
}
}
}
if (this.Buildpack == null)
{
throw new InvalidOperationException("Unable to detect a supported application type.");
}
Logger.Info("Staging task {0}: Detected buildpack {1}", this.Properties.TaskId, this.Buildpack.Name);
}
this.Properties.DetectedBuildpack = this.Buildpack.Name;
}
private void StagingStartHandler(string message, string reply, string subject)
{
StagingStartMessageRequest pmessage;
StagingInstance instance;
try
{
this.stagingTaskRegistry.Lock.EnterWriteLock();
if (this.shuttingDown)
{
return;
}
Logger.Debug("DEA Received staging message: {0}", message);
pmessage = new StagingStartMessageRequest();
try
{
pmessage.FromJsonIntermediateObject(JsonConvertibleObject.DeserializeFromJson(message));
}
catch (Exception e)
{
Logger.Error("Ignoring staging.start request. Unable to parse message. Exception: {0}", e.ToString());
return;
}
long memoryMbytes = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.MemoryMbytes != null ? pmessage.Properties.Resources.MemoryMbytes.Value : Monitoring.DefaultAppMemoryMbytes;
long diskMbytes = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.DiskMbytes != null ? pmessage.Properties.Resources.DiskMbytes.Value : Monitoring.DefaultAppDiskMbytes;
long fds = pmessage.Properties != null && pmessage.Properties.Resources != null && pmessage.Properties.Resources.FileDescriptors != null ? pmessage.Properties.Resources.FileDescriptors.Value : Monitoring.DefaultAppFds;
if (this.monitoring.MemoryReservedMbytes + memoryMbytes > this.monitoring.MaxMemoryMbytes || this.monitoring.Clients >= this.monitoring.MaxClients)
{
Logger.Info(Strings.Donothaveroomforthisclient);
return;
}
instance = this.stagingTaskRegistry.CreateStagingInstance(pmessage);
instance.Properties.MemoryQuotaBytes = memoryMbytes * 1024 * 1024;
instance.Properties.DiskQuotaBytes = diskMbytes * 1024 * 1024;
instance.Properties.FDSQuota = fds;
instance.Properties.Directory = Path.Combine(this.fileResources.StagingDir, pmessage.TaskID);
instance.Properties.TaskId = pmessage.TaskID;
instance.Properties.Reply = reply;
instance.Properties.InitializedTime = DateTime.Now;
instance.StopRequested = false;
this.monitoring.AddInstanceResources(instance);
}
finally
{
this.stagingTaskRegistry.Lock.ExitWriteLock();
}
ThreadPool.QueueUserWorkItem(delegate(object data)
{
this.StartStagingInstance(instance, pmessage);
});
}
private void StartStagingInstance(StagingInstance instance, StagingStartMessageRequest pmessage)
{
StagingWorkspace workspace = new StagingWorkspace(instance.Properties.Directory);
try
{
try
{
instance.Lock.EnterWriteLock();
instance.Properties.UseDiskQuota = this.useDiskQuota;
instance.Properties.UploadThrottleBitsps = this.uploadThrottleBitsps;
UriBuilder streamingLog = new UriBuilder();
streamingLog.Host = this.ExternalHost;
streamingLog.Scheme = "http";
streamingLog.Path = string.Format("/staging_tasks/{0}/file_path", pmessage.TaskID);
streamingLog.Query = string.Format("path={0}×tamp={1}", workspace.StagingLogSuffix, RubyCompatibility.DateTimeToEpochSeconds(DateTime.Now));
instance.Properties.StreamingLogUrl = DEAUtilities.GetHmacedUri(streamingLog.Uri.ToString(), this.directoryServerHmacKey, new string[] { "path", "timestamp" }).ToString();
instance.Workspace = workspace;
instance.Properties.TaskLog = workspace.StagingLogPath;
}
finally
{
instance.Lock.ExitWriteLock();
}
instance.AfterSetup += new StagingInstance.StagingTaskEventHandler(this.AfterStagingSetup);
Logger.Info("Started staging task {0}", instance.Properties.TaskId);
try
{
instance.SetupStagingEnvironment();
}
catch (Exception ex)
{
Logger.Error("Error setting up staging environment: ", ex.ToString());
throw ex;
}
instance.UnpackDroplet();
instance.PrepareStagingDirs();
if (File.Exists(this.logyardUidPath))
{
LogyardInstanceRequest logyardMsg = new LogyardInstanceRequest();
logyardMsg.AppGUID = pmessage.AppID;
logyardMsg.AppName = pmessage.StartMessage.Name;
logyardMsg.AppSpace = "";
logyardMsg.DockerId = instance.Properties.InstanceId;
logyardMsg.Index = -1;
logyardMsg.LogFiles = new Dictionary<string, string>() { { "staging", Path.Combine("staging", instance.Workspace.StagingLogSuffix) } };
logyardMsg.Type = "staging";
logyardMsg.RootPath = instance.Workspace.BaseDir;
string logyardId = File.ReadAllText(this.logyardUidPath).Trim();
this.deaReactor.SendLogyardNotification(logyardId, logyardMsg.SerializeToJson());
}
instance.CreatePrison();
if (pmessage.Properties.Environment != null) {
Dictionary<string, string> stagingEnvVars = this.ParseEnvironmnetVariables(pmessage.Properties.Environment);
instance.Container.User.SetUserEnvironmentVariables(stagingEnvVars);
}
instance.GetBuildpack(pmessage, this.gitPath, this.buildpacksDir);
this.stagingTaskRegistry.ScheduleSnapshotStagingState();
try
{
Logger.Info("Staging task {0}: Running compilation script", pmessage.TaskID);
this.stagingTaskRegistry.ScheduleSnapshotStagingState();
instance.CompileProcess = instance.Buildpack.StartCompile(instance.Container);
instance.Lock.EnterWriteLock();
instance.Properties.Start = DateTime.Now;
}
finally
{
if (instance.Lock.IsWriteLockHeld)
{
instance.Lock.ExitWriteLock();
}
}
}
catch (Exception ex)
{
instance.StagingException = ex;
instance.Properties.Stopped = true;
Logger.Error(ex.ToString());
}
}
public StagingInstance CreateStagingInstance(StagingStartMessageRequest message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
StagingInstance instance = new StagingInstance();
string instanceId = Credentials.GenerateSecureGuid().ToString("N");
string privateInstanceId = Credentials.GenerateSecureGuid().ToString("N") + Credentials.GenerateSecureGuid().ToString("N");
instance.StartMessage = message.StartMessage;
instance.Properties.InstanceId = instanceId;
instance.Properties.TaskId = message.TaskID;
instance.Properties.AppId = message.AppID;
instance.Properties.BuildpackCacheDownloadURI = message.BuildpackCacheDownloadURI;
instance.Properties.BuildpackCacheUploadURI = message.BuildpackCacheUploadURI;
instance.Properties.DownloadURI = message.DownloadURI;
instance.Properties.UploadURI = message.UploadURI;
if (message.Properties.Meta != null)
{
if (message.Properties.Meta.Command != null)
{
instance.Properties.MetaCommand = message.Properties.Meta.Command;
}
}
this.AddStagingInstance(instance);
return instance;
}
