private void UploadDroplet()
{
Uri uri = new Uri(this.Message.UploadURI);
Logger.Debug("Staging task {0}: Uploading droplet {1} to {2}", this.TaskId, this.workspace.StagedDroplet, this.Message.UploadURI);
DEAUtilities.HttpUploadFile(this.Message.UploadURI, new FileInfo(this.workspace.StagedDropletPath), "upload[droplet]", "application/octet-stream", uri.UserInfo);
}
public void GetBuildpack(StagingStartMessageRequest message, string gitPath, string buildpacksDir)
{
try
{
this.CreatePrison();
if (message.Properties.Buildpack != null)
{
Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.Properties.TaskId, message.Properties.Buildpack);
Directory.CreateDirectory(Path.Combine(this.Workspace.TempDir, "buildpacks"));
string buildpackPath = Path.Combine(this.Workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(message.Properties.Buildpack).LocalPath));
string command = string.Format("\"{0}\" clone --quiet --recursive {1} {2}", gitPath, message.Properties.Buildpack, buildpackPath);
Logger.Debug(command);
int success = Command.ExecuteCommand(command, this.Workspace.TempDir);
if (success != 0)
{
throw new Exception(string.Format("Failed to git clone buildpack. Exit code: {0}", success));
}
this.Buildpack = new Buildpack(buildpackPath, Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool detected = this.Buildpack.Detect(this.Prison);
if (!detected)
{
throw new Exception("Buildpack does not support this application.");
}
}
else
{
Logger.Info("Staging task {0}: Detecting buildpack", this.Properties.TaskId);
foreach (string dir in Directory.EnumerateDirectories(buildpacksDir))
{
DEAUtilities.DirectoryCopy(dir, Path.Combine(this.Workspace.TempDir, "buildpack"), true);
Buildpack bp = new Buildpack(Path.Combine(this.Workspace.TempDir, "buildpack"), Path.Combine(this.Workspace.StagedDir, "app"), this.Workspace.Cache, this.Workspace.StagingLogPath);
bool success = bp.Detect(this.Prison);
if (success)
{
this.Buildpack = bp;
break;
}
else
{
Directory.Delete(Path.Combine(this.Workspace.TempDir, "buildpack"), true);
}
}
if (this.Buildpack == null)
{
throw new Exception("Unable to detect a supported application type");
}
Logger.Info("Staging task {0}: Detected buildpack {1}", this.Properties.TaskId, this.Buildpack.Name);
}
this.Properties.DetectedBuildpack = this.Buildpack.Name;
}
finally
{
if (this.Prison.Created)
{
this.Prison.Destroy();
}
}
}
private void PackBuildpackCache()
{
Directory.CreateDirectory(this.workspace.Cache);
string tempFile = Path.ChangeExtension(this.workspace.StagedBuildpackCache, "tar");
DEAUtilities.TarDirectory(this.workspace.Cache, tempFile);
DEAUtilities.GzipFile(tempFile, this.workspace.StagedBuildpackCache);
File.Delete(tempFile);
}
private void PackApp()
{
Logger.Debug("Staging task {0}: Packing droplet {1}", this.TaskId, this.workspace.StagedDroplet);
string tempFile = Path.ChangeExtension(this.workspace.StagedDroplet, "tar");
DEAUtilities.TarDirectory(this.workspace.StagedDir, tempFile);
DEAUtilities.GzipFile(tempFile, this.workspace.StagedDroplet);
File.Delete(tempFile);
}
private void Stage()
{
string appDir = Path.Combine(this.workspace.StagedDir, "app");
string logsDir = Path.Combine(this.workspace.StagedDir, "logs");
string tmpDir = Path.Combine(this.workspace.StagedDir, "tmp");
Directory.CreateDirectory(appDir);
Directory.CreateDirectory(logsDir);
Directory.CreateDirectory(tmpDir);
DEAUtilities.DirectoryCopy(this.workspace.UnstagedDir, appDir, true);
Buildpack buildpack = null;
if (this.Message.Properties.Buildpack != null)
{
Logger.Info("Staging task {0}: Downloading buildpack from {1}", this.TaskId, this.Message.Properties.Buildpack);
Directory.CreateDirectory(Path.Combine(this.workspace.TempDir, "buildpacks"));
string buildpackPath = Path.Combine(this.workspace.TempDir, "buildpacks", Path.GetFileName(new Uri(this.Message.Properties.Buildpack).LocalPath));
string command = string.Format("\"{0}\" clone --recursive {1} {2}", this.gitExe, this.Message.Properties.Buildpack, buildpackPath);
int success = Command.ExecuteCommand(command);
if (success != 0)
{
throw new Exception("Failed to git clone buildpack");
}
buildpack = new Buildpack(buildpackPath, appDir, this.workspace.Cache, this.workspace.StagingLogPath);
bool detected = buildpack.Detect(this.prison);
if (!detected)
{
throw new Exception("Buildpack does not support this application");
}
}
else
{
Logger.Info("Staging task {0}: Detecting buildpack", this.TaskId);
foreach (string dir in Directory.EnumerateDirectories(this.buildpacksDir))
{
Buildpack bp = new Buildpack(dir, appDir, this.workspace.Cache, this.workspace.StagingLogPath);
bool success = bp.Detect(this.prison);
if (success)
{
buildpack = bp;
break;
}
}
if (buildpack == null)
{
throw new Exception("Unable to detect a supported application type");
}
Logger.Info("Staging task {0}: Detected buildpack {1}", this.TaskId, buildpack.Name);
}
Logger.Info("Staging task {0}: Running compilation script", this.TaskId);
buildpack.Compile(this.prison, this.stagingTimeout);
Logger.Info("Staging task {0}: Saving buildpackInfo", this.TaskId);
StagingInfo.SaveBuildpackInfo(Path.Combine(this.workspace.StagedDir, StagingWorkspace.StagingInfo), buildpack.Name, GetStartCommand(buildpack));
}
/// <summary>
/// Initializes a new instance of the <see cref="ApplicationBits"/> class.
/// </summary>
public ApplicationBits()
{
// Setup the Zlib here to avoid errors when extracting for the first time under an impersonated user
DEAUtilities.SetupZlib();
this.Stacks = new HashSet <string>();
if (!this.DisableDirCleanup)
{
TimerHelper.RecurringCall(CleanCacheIntervalMilliseconds, delegate { this.CleanCacheDirectory(); });
}
}
public void PrepareStagingDirs()
{
string appDir = Path.Combine(this.Workspace.StagedDir, "app");
string logsDir = Path.Combine(this.Workspace.StagedDir, "logs");
string tmpDir = Path.Combine(this.Workspace.StagedDir, "tmp");
Directory.CreateDirectory(appDir);
Directory.CreateDirectory(logsDir);
Directory.CreateDirectory(tmpDir);
DEAUtilities.DirectoryCopy(this.Workspace.UnstagedDir, appDir, true);
}
private void UnpackBuildpackCache()
{
Directory.CreateDirectory(this.workspace.Cache);
if (File.Exists(this.workspace.DownloadBuildpackCachePath))
{
Logger.Debug("Staging task {0}: Unpacking buildpack cache {1}", this.TaskId, this.workspace.DownloadBuildpackCachePath);
DEAUtilities.UnzipFile(this.workspace.Cache, this.workspace.DownloadBuildpackCachePath); // Unzip
string tarFileName = Directory.GetFiles(this.workspace.DownloadBuildpackCachePath, "*.tar")[0];
DEAUtilities.UnzipFile(this.workspace.Cache, Path.Combine(this.workspace.Cache, tarFileName)); // Untar
File.Delete(Path.Combine(this.workspace.Cache, tarFileName));
}
}
private void SaveBuildpackCache()
{
try
{
PackBuildpackCache();
}
catch
{
Logger.Debug("Staging task {0}: Cannot pack buildpack cache", this.TaskId);
return;
}
File.Copy(this.workspace.StagedBuildpackCache, this.workspace.StagedBuildpackCachePath);
Uri uri = new Uri(this.Message.BuildpackCacheUploadURI);
Logger.Debug("Staging task {0}: Uploading buildpack cache {1} to {2}", this.TaskId, this.workspace.StagedBuildpackCachePath, this.Message.BuildpackCacheUploadURI);
DEAUtilities.HttpUploadFile(this.Message.BuildpackCacheUploadURI, new FileInfo(this.workspace.StagedBuildpackCachePath), "upload[droplet]", "application/octet-stream", uri.UserInfo);
}
private void Cleanup()
{
if (this.prison.Created)
{
try
{
Logger.Info("Destroying prison for staging instance {0}", this.TaskId);
this.prison.Destroy();
}
catch (Exception ex)
{
Logger.Warning("Unable to cleanup application {0}. Exception: {1}", this.TaskId, ex.ToString());
}
}
Logger.Debug("Cleaning up directory {0}", this.workspace.BaseDir);
DEAUtilities.RemoveReadOnlyAttribute(this.workspace.BaseDir);
Directory.Delete(this.workspace.BaseDir, true);
}
public void UnpackDroplet()
{
Directory.CreateDirectory(this.Workspace.UnstagedDir);
if (File.Exists(this.Workspace.DownloadDropletPath))
{
DEAUtilities.UnzipFile(this.Workspace.UnstagedDir, this.Workspace.DownloadDropletPath);
}
else
{
throw new Exception(string.Format("Could not find file {0}", this.Workspace.DownloadDropletPath));
}
Directory.CreateDirectory(this.Workspace.Cache);
if (File.Exists(this.Workspace.DownloadBuildpackCachePath))
{
Logger.Debug("Staging task {0}: Unpacking buildpack cache {1}", this.Properties.TaskId, this.Workspace.DownloadBuildpackCachePath);
DEAUtilities.UnzipFile(this.Workspace.Cache, this.Workspace.DownloadBuildpackCachePath); // Unzip
string tarFileName = Directory.GetFiles(this.Workspace.DownloadBuildpackCachePath, "*.tar")[0];
DEAUtilities.UnzipFile(this.Workspace.Cache, Path.Combine(this.Workspace.Cache, tarFileName)); // Untar
File.Delete(Path.Combine(this.Workspace.Cache, tarFileName));
}
}
/// <summary>
/// Prepares the app directory.
/// </summary>
/// <param name="bitsFile">The bits file.</param>
/// <param name="bitsUri">The bits URI.</param>
/// <param name="hash">The sha1.</param>
/// <param name="tarZipFile">The TGZ file.</param>
/// <param name="instance">The instance.</param>
public void PrepareAppDirectory(string bitsFile, string bitsUri, string hash, string tarZipFile, DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
// What we do here, in order of preference..
// 1. Check our own staged directory.
// 2. Check shared directory from CloudController that could be mounted (bits_file)
// 3. Pull from http if needed.
string instanceDir = instance.Properties.Directory;
lock (this.stagerLock)
{
// check before downloading
if (instance.Properties.StopProcessed)
{
return;
}
if (File.Exists(tarZipFile))
{
Logger.Debug(Strings.FoundStagedBitsInLocalCache);
}
else
{
// If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version.
DateTime start = DateTime.Now;
if (File.Exists(bitsFile))
{
Logger.Debug(Strings.SharingCloudControllerStagingDirectory);
File.Copy(bitsFile, tarZipFile);
Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start);
}
else
{
Uri downloadUri = new Uri(bitsUri);
Logger.Debug(Strings.Needtodownloadappbitsfrom, downloadUri);
this.DownloadAppBits(downloadUri, hash, tarZipFile);
Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start);
}
}
// check before extracting
if (instance.Properties.StopProcessed)
{
return;
}
DateTime startStageing = DateTime.Now;
// Explode the app into its directory and optionally bind its local runtime.
Directory.CreateDirectory(instance.Properties.Directory);
DirectoryInfo deploymentDirInfo = new DirectoryInfo(instance.Properties.Directory);
DirectorySecurity deploymentDirSecurity = deploymentDirInfo.GetAccessControl();
// Owner is important to account for disk quota
deploymentDirSecurity.SetOwner(new NTAccount(instance.Properties.WindowsUserName));
deploymentDirSecurity.SetAccessRule(
new FileSystemAccessRule(
instance.Properties.WindowsUserName,
FileSystemRights.Write | FileSystemRights.Read | FileSystemRights.Delete | FileSystemRights.Modify | FileSystemRights.FullControl,
InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit,
PropagationFlags.None | PropagationFlags.InheritOnly,
AccessControlType.Allow));
// Taking ownership of a file has to be executed with restore privilege elevated privilages
using (new ProcessPrivileges.PrivilegeEnabler(Process.GetCurrentProcess(), ProcessPrivileges.Privilege.Restore))
{
deploymentDirInfo.SetAccessControl(deploymentDirSecurity);
}
// Impersonate user to cascade the owernship to every file
// Neccessary for windows disk quota
using (new UserImpersonator(instance.Properties.WindowsUserName, ".", instance.Properties.WindowsPassword, true))
{
DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip
string tarFileName = Directory.GetFiles(instanceDir, "*.tar")[0];
DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar
File.Delete(Path.Combine(instanceDir, tarFileName));
}
Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing);
}
}
/// <summary>
/// Stages the app directory.
/// </summary>
/// <param name="bitsFile">The bits file.</param>
/// <param name="bitsUri">The bits URI.</param>
/// <param name="hash">The sha1.</param>
/// <param name="tarZipFile">The TGZ file.</param>
/// <param name="instance">The instance.</param>
public void StageAppDirectory(string bitsFile, Uri bitsUri, string hash, string tarZipFile, DropletInstance instance)
{
if (instance == null)
{
throw new ArgumentNullException("instance");
}
// What we do here, in order of preference..
// 1. Check our own staged directory.
// 2. Check shared directory from CloudController that could be mounted (bits_file)
// 3. Pull from http if needed.
string instanceDir = instance.Properties.Directory;
lock (this.stagerLock)
{
// check before downloading
if (instance.Properties.StopProcessed)
{
return;
}
if (File.Exists(tarZipFile))
{
Logger.Debug(Strings.FoundStagedBitsInLocalCache);
}
else
{
// If we have a shared volume from the CloudController we can see the bits directly, just link into our staged version.
DateTime start = DateTime.Now;
if (!this.ForceHttpFileSharing && File.Exists(bitsFile))
{
Logger.Debug(Strings.SharingCloudControllerStagingDirectory);
File.Copy(bitsFile, tarZipFile);
Logger.Debug(Strings.TookXSecondsToCopyFromShared, DateTime.Now - start);
}
else
{
Logger.Debug(Strings.Needtodownloadappbitsfrom, bitsUri);
this.DownloadAppBits(bitsUri, hash, tarZipFile);
Logger.Debug(Strings.TookXSecondsToDownloadAndWrite, DateTime.Now - start);
}
}
// check before extracting
if (instance.Properties.StopProcessed)
{
return;
}
DateTime startStageing = DateTime.Now;
// Explode the app into its directory and optionally bind its local runtime.
Directory.CreateDirectory(instanceDir);
string tarFileName = Path.GetFileName(tarZipFile);
tarFileName = Path.ChangeExtension(tarFileName, ".tar");
DEAUtilities.UnzipFile(instanceDir, tarZipFile); // Unzip
DEAUtilities.UnzipFile(instanceDir, Path.Combine(instanceDir, tarFileName)); // Untar
File.Delete(Path.Combine(instanceDir, tarFileName));
Logger.Debug(Strings.TookXSecondsToStageTheApp, DateTime.Now - startStageing);
}
}
/// <summary>
/// Setups the runtimes.
/// </summary>
public void SetupRuntimes()
{
if (this.Runtimes == null || this.Runtimes.Count == 0)
{
Logger.Fatal(Strings.CannotDetermineApplicationRuntimes);
throw new InvalidOperationException(Strings.CannotDetermineApplicationRuntimes);
}
Logger.Info(Strings.Checkingruntimes);
foreach (KeyValuePair <string, DeaRuntime> kvp in this.Runtimes)
{
string name = kvp.Key;
DeaRuntime runtime = kvp.Value;
// Only enable when we succeed
runtime.Enabled = false;
// Check that we can get a version from the executable
string version_flag = string.IsNullOrEmpty(runtime.VersionArgument) ? "-v" : runtime.VersionArgument;
string expanded_exec = DEAUtilities.RunCommandAndGetOutputAndErrors("where", runtime.Executable).Trim();
expanded_exec = expanded_exec.Split(new string[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries)[0];
if (!File.Exists(expanded_exec))
{
Logger.Info(Strings.FailedExecutableNot, name, runtime.Executable, Directory.GetCurrentDirectory(), expanded_exec);
continue;
}
// java prints to stderr, so munch them both..
string version_check = DEAUtilities.RunCommandAndGetOutputAndErrors(
expanded_exec,
string.Format(CultureInfo.InvariantCulture, "{0} {1}", expanded_exec, version_flag)).Trim();
runtime.Executable = expanded_exec;
if (string.IsNullOrEmpty(runtime.Version))
{
continue;
}
// Check the version for a match
if (new Regex(runtime.Version).IsMatch(version_check))
{
// Additional checks should return true
if (!string.IsNullOrEmpty(runtime.AdditionalChecks))
{
string additional_check = DEAUtilities.RunCommandAndGetOutputAndErrors(
runtime.Executable,
string.Format(CultureInfo.InvariantCulture, "{0}", runtime.AdditionalChecks));
if (!new Regex("true").IsMatch(additional_check))
{
Logger.Info(Strings.FailedAdditionalChecks, name);
}
}
runtime.Enabled = true;
Logger.Info(Strings.RuntimeOk, name);
}
else
{
Logger.Info(Strings.FailedVersionMismatch, name, version_check);
}
}
}
private void UnpackApp()
{
Directory.CreateDirectory(this.workspace.UnstagedDir);
DEAUtilities.UnzipFile(this.workspace.UnstagedDir, this.workspace.DownloadDropletPath);
}
public void Cleanup()
{
DEAUtilities.RemoveReadOnlyAttribute(this.Workspace.BaseDir);
Directory.Delete(this.Workspace.BaseDir, true);
}
