public async Task GetPool()
{
const string PoolName = "sparkchhamosyna";
BigDataPoolsClient client = CreateClient();
BigDataPoolResourceInfo pool = await client.GetAsync(PoolName);
Assert.AreEqual(PoolName, pool.Name);
}
public PSSynapseSparkPool(BigDataPoolResourceInfo sparkPool)
: base(sparkPool?.Location, sparkPool?.Id, sparkPool?.Name, sparkPool?.Type, sparkPool?.Tags)
{
this.ProvisioningState = sparkPool.ProvisioningState;
this.AutoScale = sparkPool?.AutoScale != null ? new PSAutoScaleProperties(sparkPool.AutoScale) : null;
this.CreationDate = sparkPool?.CreationDate;
this.AutoPause = sparkPool?.AutoPause != null ? new PSAutoPauseProperties(sparkPool.AutoPause) : null;
this.SparkEventsFolder = sparkPool?.SparkEventsFolder;
this.NodeCount = sparkPool?.NodeCount;
this.LibraryRequirements = sparkPool?.LibraryRequirements != null ? new PSLibraryRequirements(sparkPool.LibraryRequirements) : null;
this.SparkVersion = sparkPool?.SparkVersion;
this.DefaultSparkLogFolder = sparkPool?.DefaultSparkLogFolder;
this.NodeSize = sparkPool?.NodeSize;
this.NodeSizeFamily = sparkPool?.NodeSizeFamily;
}
public PSSynapseSparkPool(BigDataPoolResourceInfo sparkPool)
: base(sparkPool?.Location, sparkPool?.Id, sparkPool?.Name, sparkPool?.Type, sparkPool?.Tags)
{
this.ProvisioningState = sparkPool.ProvisioningState;
this.AutoScale = sparkPool?.AutoScale != null ? new PSAutoScaleProperties(sparkPool.AutoScale) : null;
this.CreationDate = sparkPool?.CreationDate;
this.AutoPause = sparkPool?.AutoPause != null ? new PSAutoPauseProperties(sparkPool.AutoPause) : null;
this.SparkEventsFolder = sparkPool?.SparkEventsFolder;
this.NodeCount = sparkPool?.NodeCount;
this.LibraryRequirements = sparkPool?.LibraryRequirements != null ? new PSLibraryRequirements(sparkPool.LibraryRequirements) : null;
this.SparkConfigProperties = sparkPool?.SparkConfigProperties != null ? new PSSparkConfigProperties(sparkPool.SparkConfigProperties) : null;
this.SparkVersion = sparkPool?.SparkVersion;
this.DefaultSparkLogFolder = sparkPool?.DefaultSparkLogFolder;
this.NodeSize = sparkPool?.NodeSize;
this.NodeSizeFamily = sparkPool?.NodeSizeFamily;
this.IsComputeIsolationEnabled = sparkPool?.IsComputeIsolationEnabled ?? false;
this.SessionLevelPackagesEnabled = sparkPool?.SessionLevelPackagesEnabled ?? false;
this.CacheSize = sparkPool?.CacheSize ?? 0;
this.DynamicExecutorAllocation = sparkPool?.DynamicExecutorAllocation != null ? new PSDynamicExecutorAllocation(sparkPool.DynamicExecutorAllocation) : null;
this.WorkspacePackages = sparkPool?.CustomLibraries?.Select(library => new PSSynapseWorkspacePackage(library)).ToList() ?? null;
this.LastSucceededTimestamp = sparkPool?.LastSucceededTimestamp;
}
public override void ExecuteCmdlet()
{
switch (ParameterSetName)
{
case CreateByNameAndEnableAutoScaleParameterSet:
case CreateByParentObjectAndEnableAutoScaleParameterSet:
this.enableAutoScale = true;
break;
}
if (this.IsParameterBound(c => c.WorkspaceObject))
{
this.ResourceGroupName = new ResourceIdentifier(this.WorkspaceObject.Id).ResourceGroupName;
this.WorkspaceName = this.WorkspaceObject.Name;
}
if (string.IsNullOrEmpty(this.ResourceGroupName))
{
this.ResourceGroupName = this.SynapseAnalyticsClient.GetResourceGroupByWorkspaceName(this.WorkspaceName);
}
BigDataPoolResourceInfo existingSparkPool = null;
try
{
existingSparkPool = this.SynapseAnalyticsClient.GetSparkPool(this.ResourceGroupName, this.WorkspaceName, this.Name);
}
catch
{
existingSparkPool = null;
}
if (existingSparkPool != null)
{
throw new AzPSInvalidOperationException(string.Format(Resources.SynapseSparkPoolExists, this.Name, this.ResourceGroupName, this.WorkspaceName));
}
Workspace existingWorkspace = null;
try
{
existingWorkspace = this.SynapseAnalyticsClient.GetWorkspace(this.ResourceGroupName, this.WorkspaceName);
}
catch
{
existingWorkspace = null;
}
if (existingWorkspace == null)
{
throw new AzPSResourceNotFoundCloudException(string.Format(Resources.WorkspaceDoesNotExist, this.WorkspaceName));
}
LibraryRequirements libraryRequirements = null;
if (this.IsParameterBound(c => c.LibraryRequirementsFilePath))
{
var powerShellDestinationPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath(LibraryRequirementsFilePath);
libraryRequirements = new LibraryRequirements
{
Filename = Path.GetFileName(powerShellDestinationPath),
Content = this.ReadFileAsText(powerShellDestinationPath),
};
}
var createParams = new BigDataPoolResourceInfo
{
Location = existingWorkspace.Location,
Tags = TagsConversionHelper.CreateTagDictionary(this.Tag, validate: true),
NodeCount = this.enableAutoScale ? (int?)null : this.NodeCount,
NodeSizeFamily = NodeSizeFamily.MemoryOptimized,
NodeSize = NodeSize,
AutoScale = !this.enableAutoScale ? null : new AutoScaleProperties
{
Enabled = this.enableAutoScale,
MinNodeCount = AutoScaleMinNodeCount,
MaxNodeCount = AutoScaleMaxNodeCount
},
AutoPause = !EnableAutoPause ? null : new AutoPauseProperties
{
Enabled = EnableAutoPause.IsPresent,
DelayInMinutes = AutoPauseDelayInMinute
},
SparkVersion = this.SparkVersion,
LibraryRequirements = libraryRequirements
};
if (this.ShouldProcess(this.Name, string.Format(Resources.CreatingSynapseSparkPool, this.ResourceGroupName, this.WorkspaceName, this.Name)))
{
var result = new PSSynapseSparkPool(this.SynapseAnalyticsClient.CreateOrUpdateSparkPool(this.ResourceGroupName, this.WorkspaceName, this.Name, createParams));
WriteObject(result);
}
}
/// <summary>
/// Create a Big Data pool.
/// </summary>
/// <remarks>
/// Create a new Big Data pool.
/// </remarks>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The name of the resource group. The name is case insensitive.
/// </param>
/// <param name='workspaceName'>
/// The name of the workspace.
/// </param>
/// <param name='bigDataPoolName'>
/// Big Data pool name
/// </param>
/// <param name='bigDataPoolInfo'>
/// The Big Data pool to create.
/// </param>
/// <param name='force'>
/// Whether to stop any running jobs in the Big Data pool
/// </param>
/// <param name='cancellationToken'>
/// The cancellation token.
/// </param>
public static async Task <BigDataPoolResourceInfo> BeginCreateOrUpdateAsync(this IBigDataPoolsOperations operations, string resourceGroupName, string workspaceName, string bigDataPoolName, BigDataPoolResourceInfo bigDataPoolInfo, bool?force = false, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.BeginCreateOrUpdateWithHttpMessagesAsync(resourceGroupName, workspaceName, bigDataPoolName, bigDataPoolInfo, force, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Create a Big Data pool.
/// </summary>
/// <remarks>
/// Create a new Big Data pool.
/// </remarks>
/// <param name='operations'>
/// The operations group for this extension method.
/// </param>
/// <param name='resourceGroupName'>
/// The name of the resource group. The name is case insensitive.
/// </param>
/// <param name='workspaceName'>
/// The name of the workspace.
/// </param>
/// <param name='bigDataPoolName'>
/// Big Data pool name
/// </param>
/// <param name='bigDataPoolInfo'>
/// The Big Data pool to create.
/// </param>
/// <param name='force'>
/// Whether to stop any running jobs in the Big Data pool
/// </param>
public static BigDataPoolResourceInfo BeginCreateOrUpdate(this IBigDataPoolsOperations operations, string resourceGroupName, string workspaceName, string bigDataPoolName, BigDataPoolResourceInfo bigDataPoolInfo, bool?force = false)
{
return(operations.BeginCreateOrUpdateAsync(resourceGroupName, workspaceName, bigDataPoolName, bigDataPoolInfo, force).GetAwaiter().GetResult());
}
public override void ExecuteCmdlet()
{
if (this.IsParameterBound(c => c.WorkspaceObject))
{
this.ResourceGroupName = new ResourceIdentifier(this.WorkspaceObject.Id).ResourceGroupName;
this.WorkspaceName = this.WorkspaceObject.Name;
}
if (this.IsParameterBound(c => c.InputObject))
{
var resourceIdentifier = new ResourceIdentifier(this.InputObject.Id);
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
this.WorkspaceName = resourceIdentifier.ParentResource;
this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1);
this.Name = this.InputObject.Name;
}
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(this.ResourceId);
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
this.WorkspaceName = resourceIdentifier.ParentResource;
this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1);
this.Name = resourceIdentifier.ResourceName;
}
if (string.IsNullOrEmpty(this.ResourceGroupName))
{
this.ResourceGroupName = this.SynapseAnalyticsClient.GetResourceGroupByWorkspaceName(this.WorkspaceName);
}
BigDataPoolResourceInfo existingSparkPool = null;
try
{
existingSparkPool = this.SynapseAnalyticsClient.GetSparkPool(this.ResourceGroupName, this.WorkspaceName, this.Name);
}
catch
{
existingSparkPool = null;
}
if (existingSparkPool == null)
{
throw new SynapseException(string.Format(Resources.FailedToDiscoverSparkPool, this.Name, this.ResourceGroupName, this.WorkspaceName));
}
existingSparkPool.Tags = this.IsParameterBound(c => c.Tag) ? TagsConversionHelper.CreateTagDictionary(this.Tag, validate: true) : existingSparkPool.Tags;
existingSparkPool.NodeCount = this.IsParameterBound(c => c.NodeCount) ? this.NodeCount : existingSparkPool.NodeCount;
existingSparkPool.NodeSizeFamily = NodeSizeFamily.MemoryOptimized;
existingSparkPool.NodeSize = this.IsParameterBound(c => c.NodeSize) ? this.NodeSize : existingSparkPool.NodeSize;
existingSparkPool.LibraryRequirements = this.IsParameterBound(c => c.LibraryRequirementsFilePath) ? CreateLibraryRequirements() : existingSparkPool.LibraryRequirements;
if (this.IsParameterBound(c => c.EnableAutoScale) ||
this.IsParameterBound(c => c.AutoScaleMinNodeCount) ||
this.IsParameterBound(c => c.AutoScaleMaxNodeCount))
{
existingSparkPool.AutoScale = new AutoScaleProperties
{
Enabled = this.EnableAutoScale != null ? this.EnableAutoScale : existingSparkPool.AutoScale?.Enabled ?? false,
MinNodeCount = this.IsParameterBound(c => c.AutoScaleMinNodeCount) ? AutoScaleMinNodeCount : existingSparkPool.AutoScale?.MinNodeCount ?? 0,
MaxNodeCount = this.IsParameterBound(c => c.AutoScaleMaxNodeCount) ? AutoScaleMaxNodeCount : existingSparkPool.AutoScale?.MaxNodeCount ?? 0
};
}
if (this.IsParameterBound(c => c.EnableAutoPause) ||
this.IsParameterBound(c => c.AutoPauseDelayInMinute))
{
existingSparkPool.AutoPause = new AutoPauseProperties
{
Enabled = this.EnableAutoPause != null ? this.EnableAutoPause : existingSparkPool.AutoPause?.Enabled ?? false,
DelayInMinutes = this.IsParameterBound(c => c.AutoPauseDelayInMinute)
? this.AutoPauseDelayInMinute
: existingSparkPool.AutoPause?.DelayInMinutes ?? 0
};
}
if (this.ShouldProcess(this.Name, string.Format(Resources.UpdatingSynapseSparkPool, this.Name, this.ResourceGroupName, this.WorkspaceName)))
{
var result = new PSSynapseSparkPool(this.SynapseAnalyticsClient.CreateOrUpdateSparkPool(this.ResourceGroupName, this.WorkspaceName, this.Name, existingSparkPool));
WriteObject(result);
}
}
public BigDataPoolResourceInfo CreateOrUpdateSparkPool(string resourceGroupName, string workspaceName, string sparkPoolName, BigDataPoolResourceInfo createOrUpdateParams)
{
try
{
return(_synapseManagementClient.BigDataPools.CreateOrUpdate(resourceGroupName, workspaceName, sparkPoolName, createOrUpdateParams));
}
catch (ErrorContractException ex)
{
throw GetSynapseException(ex);
}
}
public override void ExecuteCmdlet()
{
if (this.IsParameterBound(c => c.WorkspaceObject))
{
this.ResourceGroupName = new ResourceIdentifier(this.WorkspaceObject.Id).ResourceGroupName;
this.WorkspaceName = this.WorkspaceObject.Name;
}
if (this.IsParameterBound(c => c.InputObject))
{
var resourceIdentifier = new ResourceIdentifier(this.InputObject.Id);
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
this.WorkspaceName = resourceIdentifier.ParentResource;
this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1);
this.Name = this.InputObject.Name;
}
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(this.ResourceId);
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
this.WorkspaceName = resourceIdentifier.ParentResource;
this.WorkspaceName = this.WorkspaceName.Substring(this.WorkspaceName.LastIndexOf('/') + 1);
this.Name = resourceIdentifier.ResourceName;
}
if (string.IsNullOrEmpty(this.ResourceGroupName))
{
this.ResourceGroupName = this.SynapseAnalyticsClient.GetResourceGroupByWorkspaceName(this.WorkspaceName);
}
BigDataPoolResourceInfo existingSparkPool = null;
try
{
existingSparkPool = this.SynapseAnalyticsClient.GetSparkPool(this.ResourceGroupName, this.WorkspaceName, this.Name);
}
catch
{
existingSparkPool = null;
}
if (existingSparkPool == null)
{
throw new AzPSResourceNotFoundCloudException(string.Format(Resources.FailedToDiscoverSparkPool, this.Name, this.ResourceGroupName, this.WorkspaceName));
}
existingSparkPool.Tags = this.IsParameterBound(c => c.Tag) ? TagsConversionHelper.CreateTagDictionary(this.Tag, validate: true) : existingSparkPool.Tags;
existingSparkPool.NodeCount = this.IsParameterBound(c => c.NodeCount) ? this.NodeCount : existingSparkPool.NodeCount;
existingSparkPool.NodeSizeFamily = NodeSizeFamily.MemoryOptimized;
existingSparkPool.NodeSize = this.IsParameterBound(c => c.NodeSize) ? this.NodeSize : existingSparkPool.NodeSize;
existingSparkPool.LibraryRequirements = this.IsParameterBound(c => c.LibraryRequirementsFilePath) ? CreateLibraryRequirements() : existingSparkPool.LibraryRequirements;
existingSparkPool.SparkConfigProperties = this.IsParameterBound(c => c.SparkConfigFilePath) ? CreateSparkConfigProperties() : existingSparkPool.SparkConfigProperties;
if (this.IsParameterBound(c => c.EnableAutoScale) ||
this.IsParameterBound(c => c.AutoScaleMinNodeCount) ||
this.IsParameterBound(c => c.AutoScaleMaxNodeCount))
{
existingSparkPool.AutoScale = new AutoScaleProperties
{
Enabled = this.EnableAutoScale != null ? this.EnableAutoScale : existingSparkPool.AutoScale?.Enabled ?? false,
MinNodeCount = this.IsParameterBound(c => c.AutoScaleMinNodeCount) ? AutoScaleMinNodeCount : existingSparkPool.AutoScale?.MinNodeCount ?? 0,
MaxNodeCount = this.IsParameterBound(c => c.AutoScaleMaxNodeCount) ? AutoScaleMaxNodeCount : existingSparkPool.AutoScale?.MaxNodeCount ?? 0
};
}
if (this.IsParameterBound(c => c.EnableAutoPause) ||
this.IsParameterBound(c => c.AutoPauseDelayInMinute))
{
existingSparkPool.AutoPause = new AutoPauseProperties
{
Enabled = this.EnableAutoPause != null ? this.EnableAutoPause : existingSparkPool.AutoPause?.Enabled ?? false,
DelayInMinutes = this.IsParameterBound(c => c.AutoPauseDelayInMinute)
? this.AutoPauseDelayInMinute
: existingSparkPool.AutoPause?.DelayInMinutes ?? int.Parse(SynapseConstants.DefaultAutoPauseDelayInMinute)
};
}
if ((!this.IsParameterBound(c => c.PackageAction) && this.IsParameterBound(c => c.Package)) ||
((this.IsParameterBound(c => c.PackageAction) && !this.IsParameterBound(c => c.Package))))
{
throw new AzPSInvalidOperationException(Resources.FailedToValidatePackageParameter);
}
if (this.IsParameterBound(c => c.PackageAction) && this.IsParameterBound(c => c.Package))
{
if (this.PackageAction == SynapseConstants.PackageActionType.Add)
{
if (existingSparkPool.CustomLibraries == null)
{
existingSparkPool.CustomLibraries = new List <LibraryInfo>();
}
existingSparkPool.CustomLibraries = existingSparkPool.CustomLibraries.Union(this.Package.Select(psPackage => new LibraryInfo
{
Name = psPackage?.Name,
Type = psPackage?.PackageType,
Path = psPackage?.Path,
ContainerName = psPackage?.ContainerName
// TODO: set uploadedTimeStamp property after upgrading SDK otherwise we will see a incorrect property value from Azure portal.
})).ToList();
}
else if (this.PackageAction == SynapseConstants.PackageActionType.Remove)
{
existingSparkPool.CustomLibraries = existingSparkPool.CustomLibraries.Where(lib => !this.Package.Any(p => lib.Path.Equals(p.Path, System.StringComparison.OrdinalIgnoreCase))).ToList();
}
}
if (this.ShouldProcess(this.Name, string.Format(Resources.UpdatingSynapseSparkPool, this.Name, this.ResourceGroupName, this.WorkspaceName)))
{
var result = new PSSynapseSparkPool(this.SynapseAnalyticsClient.CreateOrUpdateSparkPool(this.ResourceGroupName, this.WorkspaceName, this.Name, existingSparkPool));
WriteObject(result);
}
}
