public Task <EnvironmentCost> GetCost(RenderingEnvironment env, QueryTimePeriod timePeriod)
{
var cacheKey = Invariant($"REPORTING:{env.Name}/{timePeriod.From}/{timePeriod.To}");
return(_memoryCache.GetOrCreateAsync(cacheKey, Fetch));
Task <EnvironmentCost> Fetch(ICacheEntry ice)
{
ice.AbsoluteExpirationRelativeToNow = CacheResultsFor;
return(_inner.GetCost(env, timePeriod));
}
}
public async Task UpdatePool(RenderingEnvironment environment, Pool pool)
{
using (var client = await _managementClient.CreateBatchManagementClient(environment.SubscriptionId))
{
await client.Pool.UpdateAsync(environment.BatchAccount.ResourceGroupName, environment.BatchAccount.Name, pool.Name, pool);
}
// clear the cache to reload the pools.
// TODO: update the pool object in the cache
var cacheKey = CacheKeys.MakeKey(CacheKeys.PoolList, environment.Name);
_httpContextAccessor.HttpContext.Session.Remove(cacheKey);
}
public PoolDetailsModel(RenderingEnvironment environment, Pool pool, IList <PoolUsageMetric> poolUsageMetrics = null)
{
Name = pool.Name;
DisplayName = pool.DisplayName;
EnvironmentName = environment.Name;
RenderManagerType = environment.RenderManager;
// TODO: handle autoscale
DedicatedNodes = pool.ScaleSettings.FixedScale.TargetDedicatedNodes ?? 0;
LowPriorityNodes = pool.ScaleSettings.FixedScale.TargetLowPriorityNodes ?? 0;
CurrentDedicatedNodes = pool.CurrentDedicatedNodes ?? 0;
CurrentLowPriorityNodes = pool.CurrentLowPriorityNodes ?? 0;
VmSize = pool.VmSize;
// TODO: handle non-VM config
ImageReference = pool.DeploymentConfiguration.VirtualMachineConfiguration.ImageReference.Sku;
BatchAgentSku = pool.DeploymentConfiguration.VirtualMachineConfiguration.NodeAgentSkuId;
AllocationState = pool.AllocationState;
ProvisioningState = pool.ProvisioningState;
if (pool.Metadata != null)
{
AutoScaleDownIdleTimeout = pool.GetAutoScaleTimeoutInMinutes();
AutoScalePolicy = pool.GetAutoScalePolicy();
AutoScalePolicyItems.AddRange(
Enum.GetValues(typeof(AutoScalePolicy)).Cast <AutoScalePolicy>()
.Select(p => new SelectListItem(p.GetDescription(), p.ToString(), p == AutoScalePolicy)));
MinimumDedicatedNodes = pool.GetAutoScaleMinimumDedicatedNodes();
MinimumLowPriorityNodes = pool.GetAutoScaleMinimumLowPriorityNodes();
MaximumDedicatedNodes = pool.GetAutoScaleMaximumDedicatedNodes();
MaximumLowPriorityNodes = pool.GetAutoScaleMaximumLowPriorityNodes();
// TODO: This seems rather dodgy. What if there are more than one package?
SelectedPackageId = pool.Metadata.FirstOrDefault(mi => mi.Name == MetadataKeys.Package)?.Value;
SelectedGpuPackageId = pool.Metadata.FirstOrDefault(mi => mi.Name == MetadataKeys.GpuPackage)?.Value;
SelectedGeneralPackageIds = pool.Metadata.FirstOrDefault(mi => mi.Name == MetadataKeys.GeneralPackages)?.Value;
bool.TryParse(pool.Metadata.FirstOrDefault(mi => mi.Name == MetadataKeys.UseDeadlineGroups)?.Value, out var useGroups);
UseGroups = useGroups;
}
ApplicationLicenses = pool.ApplicationLicenses != null
? string.Join(", ", pool.ApplicationLicenses)
: "n/a";
PoolUsageMetrics = poolUsageMetrics;
}
// Gets all the user role assignemtns for the environment and collapses them to "Portal" roles
// that we can display for each user.
public async Task <List <UserPermission> > ListUserPermissions(RenderingEnvironment environment)
{
var environmentPermissions = await GetResourcePermissions(environment);
var userObjectIds = environmentPermissions.GetUserObjectIds();
var finalPermissions = new List <UserPermission>();
foreach (var objectId in userObjectIds)
{
var userEnvironmentPermissions = environmentPermissions.ToUserEnvironmentPermissions(objectId);
var userPermission = userEnvironmentPermissions.GetFirstUserPermission();
if (userEnvironmentPermissions.IsOwner())
{
finalPermissions.Add(new UserPermission
{
ObjectId = objectId,
Email = userPermission.Email,
Name = userPermission.Name,
Role = PortalRole.Owner.ToString(),
GraphResolutionFailure = userPermission.GraphResolutionFailure,
});
}
else if (userEnvironmentPermissions.IsPoolManager())
{
finalPermissions.Add(new UserPermission
{
ObjectId = objectId,
Email = userPermission.Email,
Name = userPermission.Name,
Role = PortalRole.PoolManager.ToString(),
GraphResolutionFailure = userPermission.GraphResolutionFailure,
});
}
else if (userEnvironmentPermissions.IsReader())
{
finalPermissions.Add(new UserPermission
{
ObjectId = objectId,
Email = userPermission.Email,
Name = userPermission.Name,
Role = PortalRole.Reader.ToString(),
GraphResolutionFailure = userPermission.GraphResolutionFailure,
});
}
}
return(finalPermissions);
}
public AddEnvironmentStep1Model(RenderingEnvironment environment)
{
if (environment != null)
{
EditMode = true;
OriginalName = environment.Name;
EnvironmentName = environment.Name;
SubscriptionId = environment.SubscriptionId;
RenderManager = environment.RenderManager;
LocationName = environment.LocationName;
SubscriptionIdLocked = (environment.ApplicationInsightsAccount != null ||
environment.KeyVault != null ||
environment.BatchAccount != null ||
environment.StorageAccount != null);
}
}
private async Task AppendQubeSetupToStartTask(
RenderingEnvironment environment,
string poolName,
StartTask startTask,
QubeConfig qubeConfig,
InstallationPackage qubePackage,
bool isWindows)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
var startTaskScriptUrl = isWindows
? GetWindowsStartTaskUrl(environment)
: GetLinuxStartTaskUrl(environment);
var uri = new Uri(startTaskScriptUrl);
var filename = uri.AbsolutePath.Split('/').Last();
var installScriptResourceFile = new ResourceFile(httpUrl: startTaskScriptUrl, filePath: filename);
resourceFiles.Add(installScriptResourceFile);
commandLine += $".\\{installScriptResourceFile.FilePath} " +
$"-qubeSupervisorIp {qubeConfig.SupervisorIp} " +
$"-workerHostGroups 'azure,{poolName}'";
if (qubePackage != null && !string.IsNullOrEmpty(qubePackage.Container))
{
resourceFiles.AddRange(await GetResourceFilesFromContainer(qubePackage.Container));
// Add qb.conf if one isn't already specified by the package
if (!resourceFiles.Any(rf => rf.FilePath.Contains("qb.conf")))
{
var qbConfResourceFile = new ResourceFile(httpUrl: _configuration["Qube:QbConf"], filePath: "qb.conf");
resourceFiles.Add(qbConfResourceFile);
}
}
else
{
// No package, lets just configure
commandLine += " -skipInstall ";
}
commandLine += ";";
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}
private void AppendTractorParamsToStartTask(
PoolConfigurationModel poolConfiguration,
RenderingEnvironment environment,
StartTask startTask,
TractorConfig tractorConfig,
InstallationPackage tractorPackage,
bool isWindows)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
if (environment.KeyVaultServicePrincipal != null)
{
commandLine += GetParameterSet(isWindows, "tenantId", environment.KeyVaultServicePrincipal.TenantId.ToString());
commandLine += GetParameterSet(isWindows, "applicationId", environment.KeyVaultServicePrincipal.ApplicationId.ToString());
commandLine += GetParameterSet(isWindows, "keyVaultCertificateThumbprint", environment.KeyVaultServicePrincipal.Thumbprint);
commandLine += GetParameterSet(isWindows, "keyVaultName", environment.KeyVault.Name);
}
var groups = $"azure,{poolConfiguration.PoolName}";
if (poolConfiguration.AdditionalGroups != null && poolConfiguration.AdditionalGroups.Any())
{
groups += $",{string.Join(',', poolConfiguration.AdditionalGroups)}";
}
if (!string.IsNullOrWhiteSpace(tractorConfig.EngineIpOrHostnameAndPort))
{
commandLine += GetParameterSet(isWindows, "engineHost", tractorConfig.EngineIpOrHostnameAndPort);
}
if (!string.IsNullOrWhiteSpace(groups))
{
commandLine += GetParameterSet(isWindows, "groups", groups);
}
if (tractorPackage != null && !string.IsNullOrEmpty(tractorPackage.Container))
{
resourceFiles.Add(GetContainerResourceFile(tractorPackage.Container, tractorPackage.PackageName));
commandLine += GetParameterSet(isWindows, "installerPath", tractorPackage.PackageName);
}
commandLine += ";";
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}
private void AppendDomainSetup(
StartTask startTask,
RenderingEnvironment environment)
{
if (environment.Domain != null && environment.Domain.JoinDomain)
{
var resourceFile = GetJoinDomainResourceFile();
startTask.ResourceFiles.Add(resourceFile);
startTask.CommandLine += $".\\{resourceFile.FilePath} " +
$"-domainName {environment.Domain.DomainName} " +
$"-domainOuPath {environment.Domain.DomainWorkerOuPath} " +
$"-tenantId {environment.KeyVaultServicePrincipal.TenantId} " +
$"-applicationId {environment.KeyVaultServicePrincipal.ApplicationId} " +
$"-keyVaultCertificateThumbprint {environment.KeyVaultServicePrincipal.Thumbprint} " +
$"-keyVaultName {environment.KeyVault.Name};";
}
}
public async Task <IList <PoolUsageResult> > GetEnvironmentUsage(RenderingEnvironment environment)
{
var result = await _queryProvider.ExecuteQuery(
environment.ApplicationInsightsAccount.ApplicationId,
environment.ApplicationInsightsAccount.ApiKey,
EnvironmentUsage);
var usage = new List <PoolUsageResult>();
if (result.Success)
{
var values = GetEnvironmentUsageMetrics(result.Results);
var poolNames = values.Select(p => p.PoolName).Distinct().ToList();
foreach (var poolName in poolNames)
{
var poolUsage = new PoolUsageResult {
PoolName = poolName
};
poolUsage.Values = values.Where(p => p.PoolName == poolName).OrderBy(p => p.Timestamp).ToList();
var lastUsage = poolUsage.Values.LastOrDefault();
if (lastUsage != null && lastUsage.Timestamp < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)))
{
// The pools last metric was > 10 minutes ago, hence the nodes are probably gone, so
// let's append a zero to cleanup the chart
poolUsage.Values.Add(new PoolUsageMetric
{
DedicatedCores = 0,
DedicatedNodes = 0,
LowPriorityCores = 0,
LowPriorityNodes = 0,
TotalCores = 0,
TotalNodes = 0,
PoolName = lastUsage.PoolName,
Timestamp = lastUsage.Timestamp.AddMinutes(1),
});
}
usage.Add(poolUsage);
}
}
else
{
Console.WriteLine($"Error querying environment {environment.Name} usage: {result.Error}");
}
return(usage);
}
public async Task <StartTask> GetDeadlineStartTask(
string poolName,
RenderingEnvironment environment,
InstallationPackage deadlinePackage,
InstallationPackage gpuPackage,
IEnumerable <InstallationPackage> generalPackages,
bool isWindows,
bool useGroups)
{
if (environment == null ||
environment.RenderManagerConfig == null ||
environment.RenderManager != RenderManagerType.Deadline)
{
throw new Exception("Wrong environment for Deadline.");
}
var resourceFiles = new List <ResourceFile>();
var startTask = new StartTask(
"",
resourceFiles,
GetEnvironmentSettings(environment),
new UserIdentity(
autoUser: new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin)),
3, // retries
true); // waitForSuccess
await AppendGpu(startTask, gpuPackage);
await AppendGeneralPackages(startTask, generalPackages);
await AppendDeadlineSetupToStartTask(
environment,
poolName,
startTask,
environment.RenderManagerConfig.Deadline,
deadlinePackage,
isWindows,
useGroups);
// Wrap all the start task command
startTask.CommandLine = $"powershell.exe -ExecutionPolicy RemoteSigned -NoProfile \"$ErrorActionPreference='Stop'; {startTask.CommandLine}\"";
return(startTask);
}
public AddEnvironmentStep4Model(RenderingEnvironment environment)
{
EnvironmentName = environment.Name;
RenderManager = environment.RenderManager;
JoinDomain = environment.Domain?.JoinDomain ?? false;
DomainName = environment.Domain?.DomainName;
DomainWorkerOuPath = environment.Domain?.DomainWorkerOuPath;
DomainJoinUsername = environment.Domain?.DomainJoinUsername;
DomainJoinPassword = environment.Domain?.DomainJoinPassword;
switch (RenderManager)
{
case RenderManagerType.Deadline:
DeadlineEnvironment = new DeadlineEnvironment
{
WindowsDeadlineRepositoryShare = environment.RenderManagerConfig?.Deadline?.WindowsRepositoryPath,
RepositoryUser = environment.RenderManagerConfig?.Deadline?.RepositoryUser,
RepositoryPassword = environment.RenderManagerConfig?.Deadline?.RepositoryPassword,
InstallDeadlineClient = environment.RenderManagerConfig?.Deadline?.LicenseServer != null,
DeadlineRegion = environment.RenderManagerConfig?.Deadline?.DeadlineRegion,
ExcludeFromLimitGroups = environment.RenderManagerConfig?.Deadline?.ExcludeFromLimitGroups,
LicenseMode = environment.RenderManagerConfig?.Deadline?.LicenseMode,
LicenseServer = environment.RenderManagerConfig?.Deadline?.LicenseServer,
RunAsService = environment.RenderManagerConfig?.Deadline?.RunAsService ?? false,
ServiceUser = environment.RenderManagerConfig?.Deadline?.ServiceUser,
ServicePassword = environment.RenderManagerConfig?.Deadline?.ServicePassword,
};
break;
case RenderManagerType.Qube610:
case RenderManagerType.Qube70:
QubeEnvironment = new QubeEnvironment
{
QubeSupervisor = environment.RenderManagerConfig?.Qube?.SupervisorIp,
};
break;
case RenderManagerType.Tractor:
TractorEnvironment = new TractorEnvironment
{
TractorSettings = environment.RenderManagerConfig?.Tractor?.TractorSettings
};
break;
}
}
private async Task AppendQubeParamsToStartTask(
PoolConfigurationModel poolConfiguration,
RenderingEnvironment environment,
StartTask startTask,
QubeConfig qubeConfig,
InstallationPackage qubePackage,
bool isWindows)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
var workerGroups = $"azure,{poolConfiguration.PoolName}";
if (poolConfiguration.AdditionalGroups != null && poolConfiguration.AdditionalGroups.Any())
{
workerGroups += $",{string.Join(',', poolConfiguration.AdditionalGroups)}";
}
commandLine += $"-qubeSupervisorIp {qubeConfig.SupervisorIp} " +
$"-workerHostGroups '{workerGroups}'";
if (qubePackage != null && !string.IsNullOrEmpty(qubePackage.Container))
{
resourceFiles.AddRange(await GetResourceFilesFromContainer(qubePackage.Container));
// Add qb.conf if one isn't already specified by the package
if (!resourceFiles.Any(rf => rf.FilePath.Contains("qb.conf")))
{
var qbConfResourceFile = new ResourceFile(httpUrl: _configuration["Qube:QbConf"], filePath: "qb.conf");
resourceFiles.Add(qbConfResourceFile);
}
}
else
{
// No package, lets just configure
commandLine += " -skipInstall ";
}
commandLine += ";";
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}
public async Task <IReadOnlyList <SelectListItem> > GetSizesSelectList(RenderingEnvironment environment)
{
var sizes = await GetSizes(environment);
string RenderMB(int?mb)
{
if (mb == null)
{
return("???");
}
if (mb >= 1024 * 1024)
{
return($"{mb / 1024.0 / 1024.0:N1} TB");
}
if (mb >= 1024)
{
return($"{mb / 1024.0:N1} GB");
}
return($"{mb} MB");
}
var list = new List <SelectListItem>();
foreach (var tier in sizes.GroupBy(v => v.Name.Substring(0, v.Name.IndexOf('_'))))
{
var group = new SelectListGroup {
Name = tier.Key
};
foreach (var sku in tier)
{
var description = $"{sku.Name.Substring(sku.Name.IndexOf('_') + 1)} ({sku.NumberOfCores} {(sku.NumberOfCores > 1 ? "vCPUs" : "vCPU")}, {RenderMB(sku.MemoryInMB)} RAM)";
list.Add(new SelectListItem(description, sku.Name)
{
Group = group
});
}
}
return(list);
}
public async Task <List <UserPermission> > ListClassicAdministrators(RenderingEnvironment environment)
{
var classicAdmins = await _azureResourceProvider.ListClassicAdministrators(environment.SubscriptionId);
var permissions = new List <UserPermission>();
foreach (var admin in classicAdmins)
{
permissions.Add(new UserPermission
{
Name = admin.EmailAddress, // The 'Name' property doesn't actually contain a name
Email = admin.EmailAddress,
ObjectId = admin.Id,
Role = admin.Role,
});
}
return(permissions);
}
public DeleteEnvironmentModel(RenderingEnvironment environment, List <GenericResource> resources = null)
{
// default to deleting the resource group
// TODO: add a flag to the env to verify if customer or us created the RG.
DeleteResourceGroup = true;
Resources = resources ?? new List <GenericResource>();
if (environment != null)
{
EnvironmentName = environment.Name;
SubscriptionId = environment.SubscriptionId;
LocationName = environment.LocationName;
ResourceGroup = environment.ResourceGroupName;
BatchAccount = environment.BatchAccount?.Name;
StorageAccount = environment.StorageAccount?.Name;
ApplicationInsights = environment.ApplicationInsightsAccount?.Name;
KeyVault = environment.KeyVault?.Name;
VNet = environment.Subnet?.VNetName;
}
}
public AddEnvironmentStep3Model(RenderingEnvironment environment)
{
EnvironmentName = environment.Name;
KeyVaultServicePrincipalCertificateName = DefaultKeyVaultCertName;
RenderManager = environment.RenderManager;
if (environment.KeyVaultServicePrincipal != null)
{
KeyVaultServicePrincipalAppId = environment.KeyVaultServicePrincipal.ApplicationId;
KeyVaultServicePrincipalObjectId = environment.KeyVaultServicePrincipal.ObjectId;
if (!string.IsNullOrEmpty(environment.KeyVaultServicePrincipal.CertificateKeyVaultName))
{
KeyVaultServicePrincipalCertificateName = environment.KeyVaultServicePrincipal.CertificateKeyVaultName;
}
}
SubscriptionId = environment.SubscriptionId;
KeyVaultName = environment.KeyVault?.Name;
}
private static UsageRequest CreateUsageRequest(RenderingEnvironment env, QueryTimePeriod period)
{
var usageRequest =
new UsageRequest(
Timeframe.Custom,
new Dataset(
Granularity.Daily,
new Dictionary <string, Aggregation>
{
{ "totalCost", new Aggregation(AggregationFunction.Sum, "PreTaxCost") }
},
new List <Grouping>
{
new Grouping("ServiceName", ColumnType.Dimension)
},
FilterExpression.Tag("environment", Operator.In, new[] { env.Id })));
usageRequest.TimePeriod = period;
return(usageRequest);
}
public async Task <StartTask> GetQubeStartTask(
string poolName,
IEnumerable <string> additionalGroups,
RenderingEnvironment environment,
InstallationPackage qubePackage,
InstallationPackage gpuPackage,
IEnumerable <InstallationPackage> generalPackages,
bool isWindows)
{
var resourceFiles = new List <ResourceFile>();
var startTask = new StartTask(
"",
resourceFiles,
GetEnvironmentSettings(environment, isWindows),
new UserIdentity(
autoUser: new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin)),
3, // retries
true); // waitForSuccess
AppendGpu(startTask, gpuPackage);
AppendDomainSetup(startTask, environment);
AppendGeneralPackages(startTask, generalPackages);
await AppendQubeSetupToStartTask(
environment,
poolName,
additionalGroups,
startTask,
environment.RenderManagerConfig.Qube,
qubePackage,
isWindows);
// Wrap all the start task command
startTask.CommandLine = $"powershell.exe -ExecutionPolicy RemoteSigned -NoProfile \"$ErrorActionPreference='Stop'; {startTask.CommandLine}\"";
return(startTask);
}
private void AppendBYOSParamsToStartTask(
PoolConfigurationModel poolConfiguration,
RenderingEnvironment environment,
StartTask startTask,
BYOSConfig byosConfig,
bool isWindows)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
if (environment.KeyVaultServicePrincipal != null)
{
commandLine += GetParameterSet(isWindows, "tenantId", environment.KeyVaultServicePrincipal.TenantId.ToString());
commandLine += GetParameterSet(isWindows, "applicationId", environment.KeyVaultServicePrincipal.ApplicationId.ToString());
commandLine += GetParameterSet(isWindows, "keyVaultCertificateThumbprint", environment.KeyVaultServicePrincipal.Thumbprint);
commandLine += GetParameterSet(isWindows, "keyVaultName", environment.KeyVault.Name);
}
if (!string.IsNullOrWhiteSpace(byosConfig.SchedulerHostnameOrIp))
{
commandLine += GetParameterSet(isWindows, "host", byosConfig.SchedulerHostnameOrIp);
}
var groups = $"azure,{poolConfiguration.PoolName}";
if (poolConfiguration.AdditionalGroups != null && poolConfiguration.AdditionalGroups.Any())
{
groups += $",{string.Join(',', poolConfiguration.AdditionalGroups)}";
}
if (!string.IsNullOrWhiteSpace(groups))
{
commandLine += GetParameterSet(isWindows, "groups", groups);
}
commandLine += ";";
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}
public AddEnvironmentStep2Model(RenderingEnvironment environment)
{
EnvironmentName = environment.Name;
SubscriptionId = environment.SubscriptionId;
LocationName = environment.LocationName;
KeyVaultName = $"{Regex.Replace(environment.Name, "/[&\\/\\\\_-]/g", "")}-kv";
RenderManager = environment.RenderManager;
if (environment.KeyVault != null)
{
KeyVaultName = environment.KeyVault.Name;
}
if (environment.BatchAccount != null)
{
NewBatchAccountName = null;
BatchAccountResourceIdLocationUrl
= $"{environment.BatchAccount.ResourceId};{environment.BatchAccount.Location};{environment.BatchAccount.Url}";
}
if (environment.StorageAccount != null)
{
NewStorageAccountName = null;
StorageAccountResourceIdAndLocation
= $"{environment.StorageAccount.ResourceId};{environment.StorageAccount.Location}";
}
if (environment.Subnet?.ResourceId != null)
{
NewVnetName = null;
SubnetResourceIdLocationAndAddressPrefix = environment.Subnet.ToString();
}
if (environment.ApplicationInsightsAccount?.ResourceId != null)
{
NewApplicationInsightsName = null;
ApplicationInsightsIdAndLocation
= $"{environment.ApplicationInsightsAccount.ResourceId};{environment.ApplicationInsightsAccount.Location}";
}
}
public async Task <PoolUsageResult> GetUsageForPool(RenderingEnvironment environment, string poolName, string vmSize)
{
var result = await _queryProvider.ExecuteQuery(
environment.ApplicationInsightsAccount.ApplicationId,
environment.ApplicationInsightsAccount.ApiKey,
GetPoolUsageQuery(poolName));
var usage = new PoolUsageResult {
PoolName = poolName
};
if (result.Success)
{
usage.Values = GetPoolUsageMetrics(result.Results);
var lastUsage = usage.Values.LastOrDefault();
if (lastUsage != null && lastUsage.Timestamp < DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(10)))
{
// The pools last metric was > 10 minutes ago, hence the nodes are probably gone, so
// let's append a zero to cleanup the chart
usage.Values.Add(new PoolUsageMetric
{
DedicatedCores = 0,
DedicatedNodes = 0,
LowPriorityCores = 0,
LowPriorityNodes = 0,
TotalCores = 0,
TotalNodes = 0,
PoolName = lastUsage.PoolName,
Timestamp = lastUsage.Timestamp.AddMinutes(1),
});
}
}
else
{
Console.WriteLine($"Error querying pool {poolName} usage: {result.Error}");
}
return(usage);
}
public async Task <IReadOnlyList <VirtualMachineSize> > GetSizes(RenderingEnvironment environment)
{
bool IsSupportedSize(VirtualMachineSize vmSize)
{
var size = vmSize.Name.Substring(vmSize.Name.IndexOf('_') + 1);
return
(size.StartsWith("D", StringComparison.Ordinal) ||
size.StartsWith("NC", StringComparison.Ordinal) ||
size.StartsWith("F", StringComparison.Ordinal) ||
size.StartsWith("H", StringComparison.Ordinal));
}
var sizes = new List <VirtualMachineSize>();
using (var computeClient = await _managementClientProvider.CreateComputeManagementClient(environment.SubscriptionId))
{
var vmSizes = await computeClient.VirtualMachineSizes.ListAsync(environment.BatchAccount.Location);
sizes.AddRange(vmSizes.Where(IsSupportedSize));
}
return(sizes);
}
private List <EnvironmentSetting> GetEnvironmentSettings(RenderingEnvironment environment)
{
var envSettings = new List <EnvironmentSetting>();
if (environment.ApplicationInsightsAccount != null)
{
envSettings.Add(new EnvironmentSetting("APP_INSIGHTS_APP_ID", environment.ApplicationInsightsAccount.ApplicationId));
envSettings.Add(new EnvironmentSetting("APP_INSIGHTS_INSTRUMENTATION_KEY", environment.ApplicationInsightsAccount.InstrumentationKey));
envSettings.Add(new EnvironmentSetting("BATCH_INSIGHTS_DOWNLOAD_URL", _configuration["BatchInsightsUrl"]));
}
var processesToWatch = $"{_configuration["BatchInsightsProcessesToWatch"]}";
if (environment.AutoScaleConfiguration != null &&
environment.AutoScaleConfiguration.SpecificProcesses != null &&
environment.AutoScaleConfiguration.SpecificProcesses.Count > 0)
{
processesToWatch += $",{string.Join(',', environment.AutoScaleConfiguration.SpecificProcesses)}";
}
envSettings.Add(new EnvironmentSetting("AZ_BATCH_MONITOR_PROCESSES", processesToWatch));
return(envSettings);
}
public async Task <StartTask> GetStartTask(
PoolConfigurationModel poolConfiguration,
RenderingEnvironment environment,
InstallationPackage renderManagerPackage,
InstallationPackage gpuPackage,
IEnumerable <InstallationPackage> generalPackages,
bool isWindows)
{
var resourceFiles = new List <ResourceFile>();
var startTask = new StartTask(
"",
resourceFiles,
GetEnvironmentSettings(environment, isWindows),
new UserIdentity(
autoUser: new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin)),
3, // retries
true); // waitForSuccess
AppendGpu(startTask, gpuPackage);
AppendDomainSetup(startTask, environment);
AppendGeneralPackages(startTask, generalPackages);
// Appends the start task script, i.e. deadline-starttask.ps1 or tractor-starttask.ps1
AppendRenderManagerStartTask(environment, startTask, isWindows);
// Appends the render manager specific parameters and package (if specified)
switch (environment.RenderManager)
{
case RenderManagerType.Deadline:
AppendDeadlineParamsToStartTask(
poolConfiguration,
environment,
startTask,
environment.RenderManagerConfig.Deadline,
renderManagerPackage,
isWindows);
break;
case RenderManagerType.OpenCue:
AppendOpenCueParamsToStartTask(
poolConfiguration,
environment,
startTask,
environment.RenderManagerConfig.OpenCue,
renderManagerPackage,
isWindows);
break;
case RenderManagerType.Qube610:
case RenderManagerType.Qube70:
await AppendQubeParamsToStartTask(
poolConfiguration,
environment,
startTask,
environment.RenderManagerConfig.Qube,
renderManagerPackage,
isWindows);
break;
case RenderManagerType.Tractor2:
AppendTractorParamsToStartTask(
poolConfiguration,
environment,
startTask,
environment.RenderManagerConfig.Tractor,
renderManagerPackage,
isWindows);
break;
case RenderManagerType.BYOS:
AppendBYOSParamsToStartTask(
poolConfiguration,
environment,
startTask,
environment.RenderManagerConfig.BYOS,
isWindows);
break;
}
// Wrap all the start task command
if (isWindows)
{
startTask.CommandLine = $"powershell.exe -ExecutionPolicy RemoteSigned -NoProfile \"$ErrorActionPreference='Stop'; {startTask.CommandLine}\"";
}
else
{
startTask.CommandLine = $"/bin/bash -c 'set -e; set -o pipefail; {startTask.CommandLine}'";
}
return(startTask);
}
private async Task AutoScaleEnvironment(RenderingEnvironment environment)
{
BatchClient client = null;
try
{
if (environment.InProgress ||
environment.BatchAccount == null)
{
return;
}
client = _batchClientAccessor.CreateBatchClient(environment);
// Pools with auto scale enabled
List <CloudPool> pools = await GetAutoScalePools(client);
if (!pools.Any())
{
// Check for pools first so we don't query app insights
// unnecessarily
return;
}
// All active nodes for the environment
var activeNodes = await _activeNodeProvider.GetActiveComputeNodes(environment);
foreach (var pool in pools)
{
// Verify pool can be resized
if (pool.State.Value != PoolState.Active ||
pool.AllocationState.Value != AllocationState.Steady)
{
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: Skipping pool State {pool.State.Value}, AllocationState {pool.AllocationState.Value}");
continue;
}
var policy = pool.GetAutoScalePolicy();
var timeout = pool.GetAutoScaleTimeoutInMinutes();
var minDedicated = pool.GetAutoScaleMinimumDedicatedNodes();
var minLowPriority = pool.GetAutoScaleMinimumLowPriorityNodes();
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"policy {policy}, " +
$"timeout {timeout}, " +
$"currentDedicated {pool.CurrentDedicatedComputeNodes.Value}, " +
$"currentLowPriority {pool.CurrentLowPriorityComputeNodes.Value}, " +
$"minimumDedicated {minDedicated}, " +
$"minimumLowPriority {minLowPriority}");
// Last acceptable active timestamp
var idleTimeCutoff = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(timeout));
// This pools nodes with CPU or whitelisted process events
var poolNodeCpuAndProcessEvents =
activeNodes.Where(an => an.PoolName == pool.Id && an.LastActive > idleTimeCutoff).ToList();
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"Nodes with process events " +
$"{poolNodeCpuAndProcessEvents.Where(e => e.TrackedProcess).Select(e => e.ComputeNodeName).Distinct().Count()}");
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"Nodes with CPU events " +
$"{poolNodeCpuAndProcessEvents.Where(e => !e.TrackedProcess).Select(e => e.ComputeNodeName).Distinct().Count()}");
// All nodes in the pool eligible for eviction.
// We ensure there's at least some CPU events lately to ensure
// app insights is running and emitting events.
var eligibleNodes = FilterNodesEligibleForEviction(pool.ListComputeNodes().ToList(), timeout)
.Where(n => poolNodeCpuAndProcessEvents.Any(pn => n.Id == pn.ComputeNodeName))
.ToList();
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"Eligible nodes " +
$"{eligibleNodes.Count}");
var activeNodeByCpuNames = new HashSet <string>();
var activeNodeByGpuNames = new HashSet <string>();
var activeNodesByProcess = new HashSet <string>();
if (policy == AutoScalePolicy.Resources || policy == AutoScalePolicy.ResourcesAndSpecificProcesses)
{
activeNodeByCpuNames = poolNodeCpuAndProcessEvents.Where(an =>
!an.TrackedProcess && // Grab nodes with CPU usage (not whitelisted)
an.CpuPercent >=
environment.AutoScaleConfiguration.MaxIdleCpuPercent) // Over the idle CPU % limit
.Select(an => an.ComputeNodeName)
.ToHashSet();
activeNodeByGpuNames = poolNodeCpuAndProcessEvents.Where(an =>
!an.TrackedProcess && // Grab nodes with GPU usage (not whitelisted)
an.GpuPercent >=
environment.AutoScaleConfiguration.MaxIdleGpuPercent) // Over the idle GPU % limit
.Select(an => an.ComputeNodeName)
.ToHashSet();
}
if (policy == AutoScalePolicy.SpecificProcesses ||
policy == AutoScalePolicy.ResourcesAndSpecificProcesses)
{
activeNodesByProcess = poolNodeCpuAndProcessEvents.Where(an => an.TrackedProcess)
.Select(an => an.ComputeNodeName)
.ToHashSet();
}
var idleNodesToShutdown = eligibleNodes.Where(
cn => !activeNodesByProcess.Contains(cn.Id) &&
!activeNodeByCpuNames.Contains(cn.Id) &&
!activeNodeByGpuNames.Contains(cn.Id)).ToList();
// Remove the idle nodes
if (idleNodesToShutdown.Any())
{
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"All Selected Idle nodes " +
$"{idleNodesToShutdown.Count}");
var dedicatedNodesToShutdown = idleNodesToShutdown.Where(n => n.IsDedicated.HasValue && n.IsDedicated.Value);
var lowPriorityNodesToShutdown = idleNodesToShutdown.Where(n => n.IsDedicated.HasValue && !n.IsDedicated.Value);
var maxDedicatedToRemove = pool.CurrentDedicatedComputeNodes.Value < minDedicated
? 0
: pool.CurrentDedicatedComputeNodes.Value - minDedicated;
var maxLowPriorityToRemove = pool.CurrentLowPriorityComputeNodes.Value < minLowPriority
? 0
: pool.CurrentLowPriorityComputeNodes.Value - minLowPriority;
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"Max nodes to remove: " +
$"maxDedicatedToRemove {maxDedicatedToRemove}, " +
$"maxLowPriorityToRemove {maxLowPriorityToRemove}");
var safeNodesToRemove = new List <ComputeNode>();
safeNodesToRemove.AddRange(dedicatedNodesToShutdown.Take(maxDedicatedToRemove));
safeNodesToRemove.AddRange(lowPriorityNodesToShutdown.Take(maxLowPriorityToRemove));
Console.WriteLine($"Autoscale for Env {environment.Name} and Pool {pool.Id}: " +
$"Removing nodes: " +
$"{safeNodesToRemove.Count}");
if (safeNodesToRemove.Any())
{
try
{
await pool.RemoveFromPoolAsync(safeNodesToRemove.Take(100));
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
}
}
catch (Exception e)
{
// TODO log
Console.WriteLine(e);
}
finally
{
if (client != null)
{
client.Dispose();
}
}
}
public ViewEnvironmentModel(
RenderingEnvironment environment,
Microsoft.Azure.Management.Batch.Models.BatchAccount batchAccount = null,
IList <PoolUsageResult> poolUsageResults = null)
{
if (environment != null)
{
EnvironmentName = environment.Name;
SubscriptionId = environment.SubscriptionId;
RenderManager = environment.RenderManager;
LocationName = environment.LocationName;
ResourceGroup = environment.ResourceGroupName;
KeyVaultName = environment.KeyVault?.Name;
KeyVaultUrl = environment.KeyVault?.Uri;
if (environment.KeyVaultServicePrincipal != null)
{
KeyVaultServicePrincipalAppId = environment.KeyVaultServicePrincipal.ApplicationId;
KeyVaultServicePrincipalObjectId = environment.KeyVaultServicePrincipal.ObjectId;
KeyVaultServicePrincipalCertificatePath = environment.KeyVaultServicePrincipal.CertificateKeyVaultName;
}
if (environment.BatchAccount != null)
{
BatchAccountName = environment.BatchAccount.Name;
BatchAccountResourceId = environment.BatchAccount.ResourceId;
BatchAccountLocation = environment.BatchAccount.Location;
BatchAccountUrl = environment.BatchAccount.Url;
}
if (environment.StorageAccount != null)
{
StorageAccountName = environment.StorageAccount.Name;
StorageAccountResourceId = environment.StorageAccount.ResourceId;
StorageAccountLocation = environment.StorageAccount.Location;
}
if (environment.Subnet != null)
{
SubnetName = environment.Subnet.Name;
SubnetVNetName = environment.Subnet.VNetName;
SubnetResourceId = environment.Subnet.ResourceId;
SubnetPrefix = environment.Subnet.AddressPrefix;
SubnetLocation = environment.Subnet.Location;
}
if (environment.ApplicationInsightsAccount?.ResourceId != null)
{
AppInsightsName = environment.ApplicationInsightsAccount.Name;
AppInsightsResourceId = environment.ApplicationInsightsAccount.ResourceId;
AppInsightsLocation = environment.ApplicationInsightsAccount.Location;
}
if (environment.AutoScaleConfiguration != null)
{
MaxIdleCpuPercent = environment.AutoScaleConfiguration.MaxIdleCpuPercent;
WhitelistedProcesses = environment.AutoScaleConfiguration.SpecificProcesses;
}
if (environment.DeletionSettings != null &&
!string.IsNullOrEmpty(environment.DeletionSettings.DeleteErrors))
{
DeleteErrors = environment.DeletionSettings.DeleteErrors;
}
if (environment.Domain != null)
{
JoinDomain = environment.Domain.JoinDomain;
DomainName = environment.Domain.DomainName;
DomainJoinUsername = environment.Domain.DomainJoinUsername;
DomainJoinPassword = environment.Domain.DomainJoinPassword;
DomainWorkerOuPath = environment.Domain.DomainWorkerOuPath;
}
if (environment.RenderManagerConfig != null)
{
if (environment.RenderManagerConfig.Deadline != null)
{
DeadlineEnvironment = new DeadlineEnvironment
{
WindowsDeadlineRepositoryShare = environment.RenderManagerConfig.Deadline.WindowsRepositoryPath,
RepositoryUser = environment.RenderManagerConfig.Deadline.RepositoryUser,
RepositoryPassword = environment.RenderManagerConfig.Deadline.RepositoryPassword,
LicenseMode = environment.RenderManagerConfig.Deadline.LicenseMode.ToString(),
LicenseServer = environment.RenderManagerConfig.Deadline.LicenseServer,
DeadlineRegion = environment.RenderManagerConfig.Deadline.DeadlineRegion,
DeadlineDatabaseCertificatePassword = environment.RenderManagerConfig.Deadline.DeadlineDatabaseCertificate?.Password,
};
}
if (environment.RenderManagerConfig.Qube != null)
{
QubeEnvironment = new QubeEnvironment
{
QubeSupervisor = environment.RenderManagerConfig.Qube.SupervisorIp,
};
}
if (environment.RenderManagerConfig.Tractor != null)
{
TractorEnvironment = new TractorEnvironment
{
TractorSettings = environment.RenderManagerConfig.Tractor.TractorSettings,
};
}
}
}
if (batchAccount != null)
{
BatchDedicatedCoreQuota = batchAccount.DedicatedCoreQuota;
BatchLowPriorityCoreQuota = batchAccount.LowPriorityCoreQuota;
BatchPoolQuota = batchAccount.PoolQuota;
}
PoolUsageResults = poolUsageResults;
}
private void AppendDeadlineParamsToStartTask(
PoolConfigurationModel poolConfiguration,
RenderingEnvironment environment,
StartTask startTask,
DeadlineConfig deadlineConfig,
InstallationPackage deadlinePackage,
bool isWindows)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
if (deadlinePackage != null && !string.IsNullOrEmpty(deadlinePackage.Container))
{
resourceFiles.Add(GetContainerResourceFile(deadlinePackage.Container, deadlinePackage.PackageName));
commandLine += GetParameterSet(isWindows, "installerPath", deadlinePackage.PackageName);
}
if (isWindows & environment.Domain != null && environment.Domain.JoinDomain)
{
commandLine += GetParameterSet(isWindows, "domainJoin");
commandLine += GetParameterSet(isWindows, "domainName", environment.Domain.DomainName);
commandLine += GetParameterSet(isWindows, "domainJoinUserName", environment.Domain.DomainJoinUsername);
if (!string.IsNullOrWhiteSpace(environment.Domain.DomainWorkerOuPath))
{
commandLine += GetParameterSet(isWindows, "domainOuPath", environment.Domain.DomainWorkerOuPath);
}
}
if (environment.KeyVaultServicePrincipal != null)
{
commandLine += GetParameterSet(isWindows, "tenantId", environment.KeyVaultServicePrincipal.TenantId.ToString());
commandLine += GetParameterSet(isWindows, "applicationId", environment.KeyVaultServicePrincipal.ApplicationId.ToString());
commandLine += GetParameterSet(isWindows, "keyVaultCertificateThumbprint", environment.KeyVaultServicePrincipal.Thumbprint);
commandLine += GetParameterSet(isWindows, "keyVaultName", environment.KeyVault.Name);
}
var repoPath = isWindows ? deadlineConfig.WindowsRepositoryPath : deadlineConfig.LinuxRepositoryPath;
if (!string.IsNullOrWhiteSpace(repoPath))
{
commandLine += GetParameterSet(isWindows, "deadlineRepositoryPath", deadlineConfig.WindowsRepositoryPath);
}
if (!string.IsNullOrEmpty(deadlineConfig.RepositoryUser))
{
commandLine += GetParameterSet(isWindows, "deadlineRepositoryUserName", deadlineConfig.RepositoryUser);
}
if (!string.IsNullOrEmpty(deadlineConfig.ServiceUser))
{
commandLine += GetParameterSet(isWindows, "deadlineServiceUserName", deadlineConfig.ServiceUser);
}
else
{
// If the Deadline slave is running under the start task context (not a service)
// then we don't want to wait for success as it will block after launching the
// Deadline launcher to prevent it being killed.
startTask.WaitForSuccess = false;
}
if (deadlineConfig.LicenseMode != null)
{
commandLine += GetParameterSet(isWindows, "deadlineLicenseMode", deadlineConfig.LicenseMode.ToString());
}
if (!string.IsNullOrEmpty(deadlineConfig.DeadlineRegion))
{
commandLine += GetParameterSet(isWindows, "deadlineRegion", deadlineConfig.DeadlineRegion);
}
if (!string.IsNullOrEmpty(deadlineConfig.LicenseServer))
{
commandLine += GetParameterSet(isWindows, "deadlineLicenseServer", deadlineConfig.LicenseServer);
}
var pools = poolConfiguration.GetDeadlinePoolsString();
if (!string.IsNullOrEmpty(pools))
{
commandLine += GetParameterSet(isWindows, "deadlinePools", pools);
}
var groups = poolConfiguration.GetDeadlineGroupsString();
if (!string.IsNullOrEmpty(groups))
{
commandLine += GetParameterSet(isWindows, "deadlineGroups", groups);
}
var limitGroups = poolConfiguration.GetDeadlineExcludeFromLimitGroupsString(deadlineConfig);
if (!string.IsNullOrWhiteSpace(limitGroups))
{
commandLine += GetParameterSet(isWindows, "excludeFromLimitGroups", limitGroups);
}
commandLine += "; ";
if (!isWindows)
{
commandLine += "wait";
}
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}
public async Task <ActionResult> Step1(AddAssetRepoStep1Model model)
{
RenderingEnvironment environment = null;
if (model.UseEnvironment)
{
if (string.IsNullOrEmpty(model.SelectedEnvironmentName))
{
ModelState.AddModelError(nameof(AddAssetRepoStep1Model.SelectedEnvironmentName), "Selected environment cannot be empty when using a environment.");
}
else
{
environment = await _environmentCoordinator.GetEnvironment(model.SelectedEnvironmentName);
if (environment == null)
{
ModelState.AddModelError(nameof(AddAssetRepoStep1Model.SelectedEnvironmentName), "Selected environment doesn't exist.");
}
}
}
if (!model.UseEnvironment)
{
if (string.IsNullOrWhiteSpace(model.SubnetResourceIdLocationAndAddressPrefix))
{
ModelState.AddModelError(nameof(AddAssetRepoStep1Model.SubnetResourceIdLocationAndAddressPrefix), "Subnet resource cannot be empty when using a VNet.");
}
if (model.SubscriptionId == null)
{
ModelState.AddModelError(nameof(AddAssetRepoStep1Model.SubscriptionId), "Subscription must be selected when not using an environment.");
}
}
if (!ModelState.IsValid)
{
model.Environments = await _environmentCoordinator.ListEnvironments();
// Validation errors, redirect back to form
return(View("Create/Step1", model));
}
// always get the repo with the originally set name
var repository = await _assetRepoCoordinator.GetRepository(model.OriginalName ?? model.RepositoryName)
?? _assetRepoCoordinator.CreateRepository(model);
if (repository.Enabled)
{
// not allowed to edit an existing enabled config
return(RedirectToAction("Overview", new { repoId = repository.Name }));
}
try
{
repository.State = AzureRenderHub.WebApp.Config.Storage.StorageState.Creating;
repository.InProgress = true;
repository.Name = model.RepositoryName;
repository.ResourceGroupName = model.RepositoryName;
repository.RepositoryType = model.RepositoryType;
repository.EnvironmentName = null;
if (model.UseEnvironment)
{
repository.SubscriptionId = environment.SubscriptionId;
repository.EnvironmentName = model.SelectedEnvironmentName;
repository.SelectedVNet = await GetAndVerifyVNet(environment.Subnet);
}
else
{
repository.SubscriptionId = model.SubscriptionId.Value;
repository.SelectedVNet = model.SelectedVirtualNetwork;
}
// pass in the original name in case we have updated it.
await _assetRepoCoordinator.UpdateRepository(repository, model.OriginalName);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error processing Step1");
ModelState.AddModelError("", $"Failed to create repository with error: {ex}");
return(View("Create/Step1", model));
}
return(RedirectToAction("Step2", new { repoId = repository.Name }));
}
private void AppendDeadlineSetupToStartTask(
RenderingEnvironment environment,
string poolName,
IEnumerable <string> additionalPools,
IEnumerable <string> additionalGroups,
StartTask startTask,
DeadlineConfig deadlineConfig,
InstallationPackage deadlinePackage,
bool isWindows,
bool useGroups)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
var startTaskScriptUrl = isWindows
? GetWindowsStartTaskUrl(environment)
: GetLinuxStartTaskUrl(environment);
var uri = new Uri(startTaskScriptUrl);
var filename = uri.AbsolutePath.Split('/').Last();
var installScriptResourceFile = new ResourceFile(httpUrl: startTaskScriptUrl, filePath: filename);
resourceFiles.Add(installScriptResourceFile);
if (isWindows)
{
commandLine += $".\\{installScriptResourceFile.FilePath} ";
}
else
{
commandLine += $"./{installScriptResourceFile.FilePath} ";
}
if (deadlinePackage != null && !string.IsNullOrEmpty(deadlinePackage.Container))
{
resourceFiles.Add(GetContainerResourceFile(deadlinePackage.Container, deadlinePackage.PackageName));
commandLine += GetParameterSet(isWindows, "installerPath", deadlinePackage.PackageName);
}
if (isWindows & environment.Domain != null && environment.Domain.JoinDomain)
{
commandLine += GetParameterSet(isWindows, "domainJoin");
commandLine += GetParameterSet(isWindows, "domainName", environment.Domain.DomainName);
commandLine += GetParameterSet(isWindows, "domainJoinUserName", environment.Domain.DomainJoinUsername);
if (!string.IsNullOrWhiteSpace(environment.Domain.DomainWorkerOuPath))
{
commandLine += GetParameterSet(isWindows, "domainOuPath", environment.Domain.DomainWorkerOuPath);
}
}
if (environment.KeyVaultServicePrincipal != null)
{
commandLine += GetParameterSet(isWindows, "tenantId", environment.KeyVaultServicePrincipal.TenantId.ToString());
commandLine += GetParameterSet(isWindows, "applicationId", environment.KeyVaultServicePrincipal.ApplicationId.ToString());
commandLine += GetParameterSet(isWindows, "keyVaultCertificateThumbprint", environment.KeyVaultServicePrincipal.Thumbprint);
commandLine += GetParameterSet(isWindows, "keyVaultName", environment.KeyVault.Name);
}
var repoPath = isWindows ? deadlineConfig.WindowsRepositoryPath : deadlineConfig.LinuxRepositoryPath;
if (!string.IsNullOrWhiteSpace(repoPath))
{
commandLine += GetParameterSet(isWindows, "deadlineRepositoryPath", deadlineConfig.WindowsRepositoryPath);
}
if (!string.IsNullOrEmpty(deadlineConfig.RepositoryUser))
{
commandLine += GetParameterSet(isWindows, "deadlineRepositoryUserName", deadlineConfig.RepositoryUser);
}
if (!string.IsNullOrEmpty(deadlineConfig.ServiceUser))
{
commandLine += GetParameterSet(isWindows, "deadlineServiceUserName", deadlineConfig.ServiceUser);
}
else
{
// If the Deadline slave is running under the start task context (not a service)
// then we don't want to wait for success as it will block after launching the
// Deadline launcher to prevent it being killed.
startTask.WaitForSuccess = false;
}
if (deadlineConfig.LicenseMode != null)
{
commandLine += GetParameterSet(isWindows, "deadlineLicenseMode", deadlineConfig.LicenseMode.ToString());
}
if (!string.IsNullOrEmpty(deadlineConfig.DeadlineRegion))
{
commandLine += GetParameterSet(isWindows, "deadlineRegion", deadlineConfig.DeadlineRegion);
}
if (!string.IsNullOrEmpty(deadlineConfig.LicenseServer))
{
commandLine += GetParameterSet(isWindows, "deadlineLicenseServer", deadlineConfig.LicenseServer);
}
var pools = useGroups ? "" : poolName;
if (additionalPools != null && additionalPools.Any())
{
pools += string.IsNullOrEmpty(pools) ? "" : ",";
pools += string.Join(',', additionalPools);
}
if (!string.IsNullOrEmpty(pools))
{
commandLine += GetParameterSet(isWindows, "deadlinePools", pools);
}
var groups = useGroups ? poolName : "";
if (additionalGroups != null && additionalGroups.Any())
{
groups += string.IsNullOrEmpty(groups) ? "" : ",";
groups += string.Join(',', additionalGroups);
}
if (!string.IsNullOrEmpty(groups))
{
commandLine += GetParameterSet(isWindows, "deadlineGroups", groups);
}
if (!string.IsNullOrWhiteSpace(deadlineConfig.ExcludeFromLimitGroups))
{
commandLine += GetParameterSet(isWindows, "excludeFromLimitGroups", deadlineConfig.ExcludeFromLimitGroups);
}
commandLine += "; ";
if (!isWindows)
{
commandLine += "wait";
}
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}
private async Task AppendDeadlineSetupToStartTask(
RenderingEnvironment environment,
string poolName,
StartTask startTask,
DeadlineConfig deadlineConfig,
InstallationPackage deadlinePackage,
bool isWindows,
bool useGroups)
{
var commandLine = startTask.CommandLine;
var resourceFiles = new List <ResourceFile>(startTask.ResourceFiles);
var startTaskScriptUrl = isWindows
? GetWindowsStartTaskUrl(environment)
: GetLinuxStartTaskUrl(environment);
var uri = new Uri(startTaskScriptUrl);
var filename = uri.AbsolutePath.Split('/').Last();
var installScriptResourceFile = new ResourceFile(httpUrl: startTaskScriptUrl, filePath: filename);
resourceFiles.Add(installScriptResourceFile);
commandLine += $".\\{installScriptResourceFile.FilePath}";
if (deadlinePackage != null && !string.IsNullOrEmpty(deadlinePackage.Container))
{
resourceFiles.Add(GetContainerResourceFile(deadlinePackage.Container, deadlinePackage.PackageName));
commandLine += $" -installerPath {deadlinePackage.PackageName}";
}
if (environment.Domain.JoinDomain)
{
commandLine += " -domainJoin";
commandLine += $" -domainName {environment.Domain.DomainName}";
commandLine += $" -domainJoinUserName {environment.Domain.DomainJoinUsername}";
if (!string.IsNullOrWhiteSpace(environment.Domain.DomainWorkerOuPath))
{
commandLine += $" -domainOuPath '{environment.Domain.DomainWorkerOuPath}'";
}
}
if (environment.KeyVaultServicePrincipal != null)
{
commandLine += $" -tenantId {environment.KeyVaultServicePrincipal.TenantId}";
commandLine += $" -applicationId {environment.KeyVaultServicePrincipal.ApplicationId}";
commandLine += $" -keyVaultCertificateThumbprint {environment.KeyVaultServicePrincipal.Thumbprint}";
commandLine += $" -keyVaultName {environment.KeyVault.Name}";
}
commandLine += $" -deadlineRepositoryPath {deadlineConfig.WindowsRepositoryPath}";
if (!string.IsNullOrEmpty(deadlineConfig.RepositoryUser))
{
commandLine += $" -deadlineRepositoryUserName {deadlineConfig.RepositoryUser}";
}
if (!string.IsNullOrEmpty(deadlineConfig.ServiceUser))
{
commandLine += $" -deadlineServiceUserName {deadlineConfig.ServiceUser}";
}
else
{
// If the Deadline slave is running under the start task context (not a service)
// then we don't want to wait for success as it will block after launching the
// Deadline launcher to prevent it being killed.
startTask.WaitForSuccess = false;
}
commandLine += $" -deadlineLicenseMode {deadlineConfig.LicenseMode.ToString()}";
if (!string.IsNullOrEmpty(deadlineConfig.DeadlineRegion))
{
commandLine += $" -deadlineRegion {deadlineConfig.DeadlineRegion}";
}
if (!string.IsNullOrEmpty(deadlineConfig.LicenseServer))
{
commandLine += $" -deadlineLicenseServer {deadlineConfig.LicenseServer}";
}
if (useGroups)
{
commandLine += $" -deadlineGroups {poolName}";
}
else
{
commandLine += $" -deadlinePools {poolName}";
}
if (!string.IsNullOrWhiteSpace(deadlineConfig.ExcludeFromLimitGroups))
{
commandLine += $" -excludeFromLimitGroups '{deadlineConfig.ExcludeFromLimitGroups}'";
}
commandLine += ";";
startTask.CommandLine = commandLine;
startTask.ResourceFiles = resourceFiles;
}