public override void ExecuteCmdlet()
{
var result = HDInsightManagementClient.GetCluster(ResourceGroupName, ClusterName);
var output = result.Select(cluster => new AzureHDInsightCluster(cluster)).ToList();
WriteObject(output, true);
}
public override void ExecuteCmdlet()
{
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(ResourceId);
this.ClusterName = resourceIdentifier.ResourceName;
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
}
if (this.IsParameterBound(c => c.InputObject))
{
this.ClusterName = this.InputObject.Name;
this.ResourceGroupName = this.InputObject.ResourceGroup;
}
if (ClusterName != null && ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
if (ShouldProcess("Disable Azure Monitor"))
{
HDInsightManagementClient.DisableAzureMonitor(ResourceGroupName, ClusterName);
WriteObject(true);
}
}
public HDInsightManagementHelper(CommonTestsFixture testMixture, MockContext context)
{
resourceManagementClient = testMixture.GetServiceClient <ResourceManagementClient>(context);
storageManagementClient = testMixture.GetServiceClient <StorageManagementClient>(context);
hdInsightManagementClient = testMixture.GetServiceClient <HDInsightManagementClient>(context);
sqlManagementClient = testMixture.GetServiceClient <SqlManagementClient>(context);
}
public override void ExecuteCmdlet()
{
var updateGatewaySettingsParameters = new UpdateGatewaySettingsParameters
{
IsCredentialEnabled = true,
UserName = HttpCredential.UserName,
Password = HttpCredential.Password.ConvertToString()
};
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(ResourceId);
this.Name = resourceIdentifier.ResourceName;
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
}
if (this.IsParameterBound(c => c.InputObject))
{
this.Name = this.InputObject.Name;
this.ResourceGroupName = this.InputObject.ResourceGroup;
}
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(Name);
}
if (ShouldProcess(Name, "set gateway http credential"))
{
HDInsightManagementClient.UpdateGatewayCredential(ResourceGroupName, Name, updateGatewaySettingsParameters);
WriteObject(new AzureHDInsightGatewaySettings(HDInsightManagementClient.GetGatewaySettings(ResourceGroupName, Name)));
}
}
public override void ExecuteCmdlet()
{
var capabilitiesResult = HDInsightManagementClient.GetProperties(Location);
var billingSpecResult = HDInsightManagementClient.ListBillingSpecs(Location);
WriteObject(new AzureHDInsightCapabilities(capabilitiesResult, billingSpecResult), true);
}
public override void ExecuteCmdlet()
{
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(ResourceId);
this.ClusterName = resourceIdentifier.ResourceName;
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
}
if (this.IsParameterBound(c => c.InputObject))
{
this.ClusterName = this.InputObject.Name;
this.ResourceGroupName = this.InputObject.ResourceGroup;
}
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
if (ShouldProcess(ClusterName))
{
AutoscaleConfigurationUpdateParameter parameter = new AutoscaleConfigurationUpdateParameter();
HDInsightManagementClient.UpdateAutoScaleConfiguration(ResourceGroupName, ClusterName, parameter);
Cluster cluster = HDInsightManagementClient.Get(ResourceGroupName, ClusterName);
WriteObject(new AzureHDInsightCluster(cluster));
}
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var result = new List <AzureHDInsightRuntimeScriptActionDetail>();
if (ScriptExecutionId.HasValue)
{
var executionDetailResponse = HDInsightManagementClient.GetScriptExecutionDetail(ResourceGroupName, ClusterName, ScriptExecutionId.Value);
if (executionDetailResponse != null && executionDetailResponse.RuntimeScriptActionDetail != null)
{
result.Add(new AzureHDInsightRuntimeScriptActionDetail(executionDetailResponse.RuntimeScriptActionDetail));
}
}
else
{
var executionHistory = HDInsightManagementClient.ListScriptExecutionHistory(ResourceGroupName, ClusterName);
if (executionHistory != null)
{
result.AddRange(executionHistory.Select(h => new AzureHDInsightRuntimeScriptActionDetail(h)));
}
}
WriteObject(result, true);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var scriptAction = new RuntimeScriptAction
{
Name = Name,
Parameters = Parameters,
Roles = NodeTypes.Select(n => n.ToString()).ToList(),
Uri = Uri,
ApplicationName = ApplicationName
};
var scriptActions = new List <RuntimeScriptAction> {
scriptAction
};
var executeScriptActionParameters = new ExecuteScriptActionParameters
{
ScriptActions = scriptActions,
PersistOnSuccess = PersistOnSuccess.IsPresent
};
var operationResource = HDInsightManagementClient.ExecuteScriptActions(ResourceGroupName, ClusterName, executeScriptActionParameters);
WriteObject(new AzureHDInsightRuntimeScriptActionOperationResource(scriptAction, operationResource));
}
public void TestExecuteScriptActionsCustomizations()
{
TestDelegatingHandler handler = new TestDelegatingHandler();
HDInsightManagementClient client = GetHDInsightUnitTestingClient(handler);
List <RuntimeScriptAction> scriptActions = new List <RuntimeScriptAction>
{
new RuntimeScriptAction
{
Name = "name",
Parameters = "params",
Roles = new List <string> {
"headnode", "workernode"
},
Uri = "http://foo.bar"
}
};
client.Clusters.BeginExecuteScriptActions(ResourceGroupName, ClusterName, new ExecuteScriptActionParameters
{
PersistOnSuccess = true,
ScriptActions = scriptActions
});
client.Clusters.BeginExecuteScriptActions(ResourceGroupName, ClusterName, scriptActions, true);
Assert.Equal(handler.Requests[0], handler.Requests[1]);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var result = new List <AzureHDInsightRuntimeScriptAction>();
var persistedScripts = HDInsightManagementClient.ListPersistedScripts(ResourceGroupName, ClusterName);
if (persistedScripts != null)
{
if (string.IsNullOrEmpty(Name))
{
result.AddRange(persistedScripts.Select(p => new AzureHDInsightRuntimeScriptAction(p)));
}
else
{
var filteredScripts = persistedScripts.Where(p => string.Equals(p.Name, Name, StringComparison.OrdinalIgnoreCase))
.Select(p => new AzureHDInsightRuntimeScriptAction(p));
result.AddRange(filteredScripts);
}
}
WriteObject(result, true);
}
private string GetStorageAccountKey(string resourceGroupName, string clusterName)
{
string storageAccountKey = null;
string errorMessage = "Fails to retrieve storage account key. Please specify DefaultStorageAccountKey explicitly.";
const string AuthorizationFailedCode = "AuthorizationFailed";
try
{
IDictionary <string, string> coreSiteClusterConfiguration;
HDInsightManagementClient.ListConfigurations(resourceGroupName, clusterName).Configurations.TryGetValue(ConfigurationKey.CoreSite, out coreSiteClusterConfiguration);
coreSiteClusterConfiguration?.TryGetValue(Constants.ClusterConfiguration.StorageAccountKeyPrefix + DefaultStorageAccountName, out storageAccountKey);
}
catch (CloudException cloudEx)
{
if (cloudEx.Error.Code == AuthorizationFailedCode)
{
errorMessage = "Insufficient permissions to retrieve storage account key. Please specify DefaultStorageAccountKey explicitly.";
}
}
catch (Exception ex)
{
errorMessage = errorMessage + " Reason: " + ex.Message;
}
if (storageAccountKey == null)
{
throw new CloudException(errorMessage);
}
return(storageAccountKey);
}
protected override void ProcessRecord()
{
var rdpParams = new RDPSettingsParameters
{
OsProfile = new OsProfile
{
WindowsOperatingSystemProfile = new WindowsOperatingSystemProfile
{
RdpSettings = new RdpSettings
{
UserName = RdpCredential.UserName,
Password = RdpCredential.Password.ConvertToString(),
ExpiryDate = RdpAccessExpiry
}
}
}
};
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
HDInsightManagementClient.ConfigureRdp(ResourceGroupName, ClusterName, rdpParams);
}
public override void ExecuteCmdlet()
{
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(ResourceId);
this.ClusterName = resourceIdentifier.ResourceName;
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
}
if (this.IsParameterBound(c => c.InputObject))
{
this.ClusterName = this.InputObject.Name;
this.ResourceGroupName = this.InputObject.ResourceGroup;
}
if (ClusterName != null && ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var result = HDInsightManagementClient.GetHosts(ResourceGroupName, ClusterName)
.Select(entry => new AzureHDInsightHostInfo(entry)).ToList();
WriteObject(result, true);
}
public override void ExecuteCmdlet()
{
AzureHDInsightAutoscale autoscaleConfiguration = null;
if (this.IsParameterBound(c => c.InputObject))
{
autoscaleConfiguration = InputObject?.ComputeProfile?.Roles?.FirstOrDefault(role => role.Name.Equals("workernode"))?.AutoscaleConfiguration;
}
else
{
if (this.IsParameterBound(c => c.ResourceId))
{
var resourceIdentifier = new ResourceIdentifier(ResourceId);
this.ClusterName = resourceIdentifier.ResourceName;
this.ResourceGroupName = resourceIdentifier.ResourceGroupName;
}
if (ClusterName != null && ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var cluster = HDInsightManagementClient.Get(ResourceGroupName, ClusterName);
var autoscale = Utils.ExtractWorkerNode(cluster)?.AutoscaleConfiguration;
autoscaleConfiguration = autoscale != null ? new AzureHDInsightAutoscale(autoscale) : null;
}
WriteObject(autoscaleConfiguration);
}
protected AzureHDInsightDefaultStorageAccount GetDefaultStorageAccount(string resourceGroupName, string clusterName)
{
var result = HDInsightManagementClient.GetCluster(resourceGroupName, clusterName);
if (result == null || result.Count == 0)
{
throw new CloudException(string.Format("Couldn't find cluster {0}", clusterName));
}
var cluster = result.FirstOrDefault();
var coreSiteConfiguration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, ConfigurationKey.CoreSite);
var clusterIdentityConfiguration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, ConfigurationKey.ClusterIdentity);
var DefaultStorageAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
cluster.Properties.ClusterVersion,
coreSiteConfiguration,
clusterIdentityConfiguration
);
if (DefaultStorageAccount == null)
{
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", clusterName));
}
return(DefaultStorageAccount);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
switch (ParameterSetName)
{
case SetByAzureHDInsightHostInfoParameterSet:
{
if (this.IsParameterBound(c => c.AzureHDInsightHostInfo))
{
Name = AzureHDInsightHostInfo.Select(entry => entry.name).ToArray();
}
break;
}
default:
break;
}
string hosts = string.Join(",", Name);
if (ShouldProcess(hosts, "restart host"))
{
HDInsightManagementClient.RestartHosts(ResourceGroupName, ClusterName, Name);
if (PassThru.IsPresent)
{
WriteObject(true);
}
}
}
protected override void ProcessRecord()
{
parameters.UserName = HttpCredential.UserName;
parameters.Password = HttpCredential.Password.ConvertToString();
if (RdpCredential != null)
{
parameters.RdpUsername = RdpCredential.UserName;
parameters.RdpPassword = RdpCredential.Password.ConvertToString();
}
if (OSType == OSType.Linux)
{
parameters.SshUserName = SshCredential.UserName;
if (!string.IsNullOrEmpty(SshCredential.Password.ConvertToString()))
{
parameters.SshPassword = SshCredential.Password.ConvertToString();
}
if (!string.IsNullOrEmpty(SshPublicKey))
{
parameters.SshPublicKey = SshPublicKey;
}
}
foreach (
var storageAccount in
AdditionalStorageAccounts.Where(
storageAccount => !parameters.AdditionalStorageAccounts.ContainsKey(storageAccount.Key)))
{
parameters.AdditionalStorageAccounts.Add(storageAccount.Key, storageAccount.Value);
}
foreach (var config in Configurations.Where(config => !parameters.Configurations.ContainsKey(config.Key)))
{
parameters.Configurations.Add(config.Key, config.Value);
}
foreach (var action in ScriptActions.Where(action => parameters.ScriptActions.ContainsKey(action.Key)))
{
parameters.ScriptActions.Add(action.Key,
action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
}
if (OozieMetastore != null)
{
var metastore = OozieMetastore;
parameters.OozieMetastore = new Metastore(metastore.SqlAzureServerName, metastore.DatabaseName, metastore.Credential.UserName, metastore.Credential.Password.ConvertToString());
}
if (HiveMetastore != null)
{
var metastore = HiveMetastore;
parameters.OozieMetastore = new Metastore(metastore.SqlAzureServerName, metastore.DatabaseName, metastore.Credential.UserName, metastore.Credential.Password.ConvertToString());
}
var cluster = HDInsightManagementClient.CreateNewCluster(ResourceGroupName, ClusterName, parameters);
if (cluster != null)
{
WriteObject(new AzureHDInsightCluster(cluster.Cluster));
}
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
HDInsightManagementClient.DeletePersistedScript(ResourceGroupName, ClusterName, Name);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
HDInsightManagementClient.PromoteScript(ResourceGroupName, ClusterName, ScriptExecutionId);
}
private HDInsightManagementClient GetHDInsightUnitTestingClient(TestDelegatingHandler handler)
{
ServiceClientCredentials cred = new BasicAuthenticationCredentials();
HDInsightManagementClient client = new HDInsightManagementClient(cred, handler)
{
SubscriptionId = Guid.NewGuid().ToString()
};
return(client);
}
public override void ExecuteCmdlet()
{
var httpParams = new HttpSettingsParameters
{
HttpUserEnabled = false
};
HDInsightManagementClient.ConfigureHttp(ResourceGroupName, ClusterName, httpParams);
WriteObject(HDInsightManagementClient.GetConnectivitySettings(ResourceGroupName, ClusterName));
}
public void TestCreateCustomization()
{
string testName = "TestCreateCustomization";
TestDelegatingHandler handler = new TestDelegatingHandler();
HDInsightManagementClient client = GetHDInsightUnitTestingClient(handler);
client.Clusters.BeginCreating(ResourceGroupName, ClusterName, ClusterCreateParametersHelpers.GetCustomCreateParametersIaas(testName));
client.Clusters.BeginCreate(ResourceGroupName, ClusterName, ClusterCreateParametersHelpers.GetIaasClusterSpec(testName.ToLowerInvariant()));
Assert.Equal(handler.Requests[0], handler.Requests[1]);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(Name);
}
var operationResource = HDInsightManagementClient.GetOMS(ResourceGroupName, Name);
WriteObject(new AzureHDInsightOMS(operationResource));
}
protected override void ProcessRecord()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var result = HDInsightManagementClient.DeleteCluster(ResourceGroupName, ClusterName);
WriteObject(result, true);
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(Name);
}
var clusterMonitoringResource = HDInsightManagementClient.GetMonitoring(ResourceGroupName, Name);
WriteObject(new AzureHDInsightMonitoring(clusterMonitoringResource));
}
public override void ExecuteCmdlet()
{
HDInsightManagementClient.ResizeCluster(ResourceGroupName, ClusterName, resizeParams);
var cluster = HDInsightManagementClient.GetCluster(ResourceGroupName, ClusterName);
if (cluster != null)
{
WriteObject(new AzureHDInsightCluster(cluster.First()));
}
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
}
var result = HDInsightManagementClient.DeleteCluster(ResourceGroupName, ClusterName);
WriteObject(result, true);
}
public override void ExecuteCmdlet()
{
var httpParams = new HttpSettingsParameters
{
HttpUserEnabled = true,
HttpUsername = HttpCredential.UserName,
HttpPassword = HttpCredential.Password.ConvertToString()
};
HDInsightManagementClient.ConfigureHttp(ResourceGroupName, ClusterName, httpParams);
WriteObject(HDInsightManagementClient.GetConnectivitySettings(ResourceGroupName, ClusterName));
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(Name);
}
if (ShouldProcess("Disable Operations Management Suite"))
{
var operationResource = HDInsightManagementClient.DisableOMS(ResourceGroupName, Name);
WriteObject(operationResource);
}
}
public override void ExecuteCmdlet()
{
if (ResourceGroupName == null)
{
ResourceGroupName = GetResourceGroupByAccountName(Name);
}
if (ShouldProcess("Disable Monitoring"))
{
HDInsightManagementClient.DisableMonitoring(ResourceGroupName, Name);
WriteObject(true);
}
}
public static void WaitForClusterToMoveToRunning(string resourceGroup, string dnsName, HDInsightManagementClient hdInsightClient)
{
System.TimeSpan timeout = System.TimeSpan.FromMinutes(10);
var stopwatch = System.Diagnostics.Stopwatch.StartNew();
bool createError = false;
do
{
var cluster = hdInsightClient.Clusters.Get(resourceGroup, dnsName);
if (cluster.Cluster.Properties.ClusterState == "Error")
{
createError = true;
break;
}
if (cluster.Cluster.Properties.ClusterState == "Running") { return; }
System.Threading.Thread.Sleep(2000);
}
while (stopwatch.Elapsed < timeout);
Xunit.Assert.True(!createError);
}
public void ResizeWithOngoingResize(HDInsightManagementClient client, string resourceGroup, string dnsname)
{
var cluster = client.Clusters.Get(resourceGroup, dnsname);
var role = cluster.Cluster.Properties.ComputeProfile.Roles.FirstOrDefault(r => r.Name == "workernode");
Assert.NotNull(role);
var targetInstanceCount = role.TargetInstanceCount;
var resizeParams = new ClusterResizeParameters
{
TargetInstanceCount = targetInstanceCount + 1
};
client.Clusters.BeginResizing(resourceGroup, dnsname, resizeParams);
try
{
client.Clusters.Resize(resourceGroup, dnsname, resizeParams);
}
catch
{
}
}
public void ResizeWithMissingTargetInstanceCount(HDInsightManagementClient client, string resourceGroup, string dnsname)
{
var resizeParams = new ClusterResizeParameters();
try
{
client.Clusters.Resize(resourceGroup, dnsname, resizeParams);
}
catch (CloudException ex)
{
Assert.Equal(ex.Response.StatusCode, HttpStatusCode.BadRequest);
}
}
public static int Main(string[] args)
{
var command = string.Empty;
try
{
if (args.Length == 0)
{
Console.Error.WriteLine("No input args, you should provide at least one arg");
ShowHelp();
return 9999;
}
command = args[0];
if ((command == "help") || (command == "/?"))
{
ShowHelp();
return -2;
}
config = new Config(args);
config.PrintRunConfiguration();
if (!config.SilentMode)
{
Logger.InfoFormat("\n========== Microsoft HDInsight Management Command Line Tool ==========\n");
Logger.InfoFormat("Command: {0}", command);
Logger.InfoFormat("TimeoutPeriodInMinutes: {0}, OperationPollIntervalInSeconds: {1}. Overridable through command line or config file\n",
config.TimeoutPeriodInMinutes, config.OperationPollIntervalInSeconds);
}
Logger.InfoFormat("Getting Azure ActiveDirectory Token from {0}", config.AzureActiveDirectoryUri);
AuthenticationResult token = null;
AuthenticationContext ac = new AuthenticationContext(config.AzureActiveDirectoryUri + config.AzureActiveDirectoryTenantId, true);
var promptBehavior = PromptBehavior.Auto;
while (token == null)
{
try
{
Logger.DebugFormat("Acquring token from {0} with behavior: {1}", config.AzureActiveDirectoryUri + config.AzureActiveDirectoryTenantId, promptBehavior);
token = ac.AcquireToken(config.AzureManagementUri, config.AzureActiveDirectoryClientId,
new Uri(config.AzureActiveDirectoryRedirectUri), promptBehavior);
}
catch(Exception ex)
{
Logger.ErrorFormat("An error occurred while acquiring token from Azure Active Directory, will retry. Exception:\r\n{0}", ex.ToString());
promptBehavior = PromptBehavior.RefreshSession;
}
}
Logger.InfoFormat("Acquired Azure ActiveDirectory Token for User: {0} with Expiry: {1}", token.UserInfo.GivenName, token.ExpiresOn);
tokenCloudCredentials = new TokenCloudCredentials(config.SubscriptionId, token.AccessToken);
Logger.InfoFormat("Connecting to AzureResourceManagementUri endpoint at {0}", config.AzureResourceManagementUri);
HttpClient azureResourceProviderHandlerHttpClient = null;
if (!AzureResourceProviderHandler.HDInsightResourceProviderNamespace.Equals(config.AzureResourceProviderNamespace, StringComparison.OrdinalIgnoreCase))
{
azureResourceProviderHandlerHttpClient = new HttpClient(new AzureResourceProviderHandler(config.AzureResourceProviderNamespace));
hdInsightManagementClient = new HDInsightManagementClient(tokenCloudCredentials,
new Uri(config.AzureResourceManagementUri), azureResourceProviderHandlerHttpClient);
}
else
{
hdInsightManagementClient = new HDInsightManagementClient(tokenCloudCredentials, new Uri(config.AzureResourceManagementUri));
}
pollInterval = TimeSpan.FromSeconds(config.OperationPollIntervalInSeconds);
timeout = TimeSpan.FromMinutes(config.TimeoutPeriodInMinutes);
totalStopWatch = new Stopwatch();
totalStopWatch.Start();
switch (command)
{
case "l":
case "list":
{
Logger.InfoFormat("SubscriptionId: {0} - Getting cluster information", config.SubscriptionId);
var clusters = hdInsightManagementClient.Clusters.List();
int i = 1;
foreach (var cluster in clusters)
{
Logger.InfoFormat("Cluster {0}: {1}", i++, cluster.Name);
Logger.InfoFormat(ClusterToString(cluster));
}
break;
}
case "ls":
case "listspecific":
{
Logger.InfoFormat("Cluster: {0} - Getting details", config.ClusterDnsName);
var cluster = hdInsightManagementClient.Clusters.Get(config.ResourceGroupName, config.ClusterDnsName).Cluster;
Logger.InfoFormat(ClusterToString(cluster));
Logger.InfoFormat("Cluster Configurations:\r\n{0}", GetClusterConfigurations());
break;
}
case "lsrc":
case "listspecificresumecreate":
case "lsrd":
case "listspecificresumedelete":
{
if (command.Contains("lsrc") || command.Contains("create"))
{
MonitorCreate(config.ResourceGroupName, config.ClusterDnsName);
}
else if (command.Contains("lsrd") || command.Contains("delete"))
{
MonitorDelete(config.ResourceGroupName, config.ClusterDnsName);
}
break;
}
case "c":
case "create":
{
foreach (var asv in config.AdditionalStorageAccounts)
{
if (!asv.Name.EndsWith(".net", StringComparison.OrdinalIgnoreCase))
Logger.InfoFormat("WARNING - ASV AccountName: {0} does not seem to have a valid FQDN. " +
"Please ensure that you use the full blob endpoint url else your cluster creation will fail.",
asv.Name);
}
CreateResourceGroup(config.AzureResourceManagementUri,
config.AzureResourceProviderNamespace,
config.ResourceGroupName, config.ClusterLocation);
Create(config.SubscriptionId, config.ResourceGroupName, config.ClusterDnsName, config.ClusterLocation,
config.DefaultStorageAccount, config.ClusterSize,
config.ClusterUsername, config.ClusterPassword, config.HDInsightVersion,
config.ClusterType, config.OSType,
config.AdditionalStorageAccounts, config.SqlAzureMetastores,
config.VirtualNetworkId, config.SubnetName,
config.HeadNodeSize, config.WorkerNodeSize, config.ZookeeperSize);
break;
}
case "rs":
case "resize":
{
Resize(config.ResourceGroupName, config.ClusterDnsName, config.ClusterSize);
break;
}
case "rdpon":
{
EnableRdp(config.ClusterDnsName, config.ClusterLocation,
config.RdpUsername, config.RdpPassword,
DateTime.Now.AddDays(int.Parse(config.RdpExpirationInDays)));
break;
}
case "rdpoff":
{
DisableRdp(config.ClusterDnsName, config.ClusterLocation);
break;
}
case "d":
case "delete":
{
Delete(config.ResourceGroupName, config.ClusterDnsName);
break;
}
case "derr":
case "deleteerror":
{
Logger.InfoFormat("SubId: {0} - Deleting all clusters in error or unknown state", config.SubscriptionId);
var clustersResponse = hdInsightManagementClient.Clusters.List();
var errorClustersList = new List<Cluster>();
foreach (var cluster in clustersResponse.Clusters)
{
Logger.InfoFormat("Found: {0}, State: {1}, CreatedDate: {2}",
cluster.Name, cluster.Properties.ClusterState, cluster.Properties.CreatedDate);
if (cluster.Properties.ProvisioningState == HDInsightClusterProvisioningState.Failed ||
cluster.Properties.ProvisioningState == HDInsightClusterProvisioningState.Canceled ||
String.Compare(cluster.Properties.ClusterState, "Error", StringComparison.OrdinalIgnoreCase) == 0 ||
String.Compare(cluster.Properties.ClusterState, "Unknown", StringComparison.OrdinalIgnoreCase) == 0)
{
errorClustersList.Add(cluster);
}
}
Logger.InfoFormat("Clusters found: {0}, Clusters in Error/Unknown state: {1}", clustersResponse.Clusters.Count, errorClustersList.Count);
var deleteCount = 0;
foreach (var errorCluster in errorClustersList)
{
Delete(config.ResourceGroupName, errorCluster.Name);
deleteCount++;
}
Logger.InfoFormat("Clusters deleted: {0}", deleteCount);
break;
}
case "dstale":
case "deletestale":
case "dall":
case "deleteall":
case "dallprefix":
case "deleteallprefix":
{
if (!ConfirmOperation(String.Format("SubId: {0} - Are you sure you want delete all the clusters (Clusters that were created more than {1} hours ago)? This operation cannot be undone.",
config.SubscriptionId, config.DeleteCutoffPeriodInHours)))
{
break;
}
var clustersResponse = hdInsightManagementClient.Clusters.List();
var deleteClustersList = new List<Cluster>();
var currTime = DateTime.UtcNow;
var cutoffTime = DateTime.UtcNow.AddHours(-config.DeleteCutoffPeriodInHours);
Logger.InfoFormat(
Environment.NewLine + String.Format("Current UTC time: {0}, Cut-off time: {1}", currTime.ToString(), cutoffTime.ToString()) +
Environment.NewLine + String.Format("Total Clusters: {0}", clustersResponse.Clusters.Count) +
Environment.NewLine + "Searching for Clusters...");
foreach (var cluster in clustersResponse)
{
if (cluster.Properties.CreatedDate < cutoffTime)
{
Logger.InfoFormat("Name: {0}, State: {1}, CreatedDate: {2}",
cluster.Name, cluster.Properties.ProvisioningState, cluster.Properties.CreatedDate);
deleteClustersList.Add(cluster);
}
}
Logger.InfoFormat("Clusters to be deleted: {0}", deleteClustersList.Count);
var deleteCount = 0;
if (deleteClustersList.Count == 0 || !ConfirmOperation())
{
break;
}
else
{
foreach (var deleteCluster in deleteClustersList)
{
try
{
Delete(config.ResourceGroupName, deleteCluster.Name);
deleteCount++;
}
catch (Exception ex)
{
Logger.InfoFormat("SubId: {0} - Cluster: {1} - Error encountered during Delete:\r\n{2}",
config.SubscriptionId, deleteCluster.Name, ex.ToString());
}
}
}
Logger.InfoFormat("Clusters deleted: {0}", deleteCount);
break;
}
case "gsl":
case "getsupportedlocations":
{
GetSupportedLocations(config.ClusterLocation);
break;
}
case "gc":
case "getcapabilties":
{
GetCapabilities(config.SubscriptionId);
break;
}
default:
{
Logger.InfoFormat(string.Format("Command '{0}' is incorrect.", command));
ShowHelp();
throw new Exception();
}
}
Logger.InfoFormat("Total time taken for this command ({0}): {1:0.00} secs", command, totalStopWatch.Elapsed.TotalSeconds);
totalStopWatch.Stop();
}
catch (Exception ex)
{
Logger.Error(String.Format("Command '{0}' failed. Error Information:", command ?? "null") , ex);
return -1;
}
finally
{
if (Debugger.IsAttached && !config.SilentMode)
{
Logger.Info("Press any key to exit...");
Console.ReadKey();
}
}
return 0;
}
private ClusterCreateParameters CreateClusterToValidateScriptActions(string resourceGroup, string dnsName, HDInsightManagementClient client)
{
var clusterCreateParams = GetClusterSpecHelpers.GetCustomCreateParametersIaas();
client.Clusters.Create(resourceGroup, dnsName, clusterCreateParams);
HDInsightManagementTestUtilities.WaitForClusterToMoveToRunning(resourceGroup, dnsName, client);
string storageAccountName =clusterCreateParams.DefaultStorageAccountName.Split('.')[0];
//upload only in record mode, storageaccount will be empty in playback mode
if (!string.IsNullOrEmpty(storageAccountName))
{
//upload failing script to the cluster default storage account
var creds = new StorageCredentials(storageAccountName, clusterCreateParams.DefaultStorageAccountKey);
var storageAccount = new CloudStorageAccount(creds, true);
var blobClient = storageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference(FailingScriptLocationContainer);
container.CreateIfNotExists();
var blockBlob = container.GetBlockBlobReference("failingscriptaction.sh");
using (var fileStream = System.IO.File.OpenRead(@"TestData/FailingScriptAction.sh"))
{
blockBlob.UploadFromStream(fileStream);
}
}
return clusterCreateParams;
}
