public HDInsightClusterCreateException(ClusterDetails clusterDetails)
: this(clusterDetails != null ? string.Format(CultureInfo.InvariantCulture,
"Unable to complete the cluster create operation. Operation failed with code '{0}'. Cluster left behind state: '{1}'. Message: '{2}'.",
clusterDetails.Error.HttpCode,
clusterDetails.StateString ?? "NULL",
clusterDetails.Error.Message ?? "NULL") : DefaultExceptionMessage,
null)
{
this.hdiClusterDetails = clusterDetails;
}
/// <summary>
/// Initializes a new instance of the HDInsightApplicationHistoryClient class.
/// </summary>
/// <param name="cluster">
/// Details about the cluster against which the client should be created.
/// </param>
/// <param name="timeout">
/// The timeout value to use for operations performed by this client.
/// </param>
/// <param name="customUserAgent">
/// A tag to help identify the client/user.
/// </param>
internal HDInsightApplicationHistoryClient(ClusterDetails cluster, TimeSpan timeout, string customUserAgent)
{
cluster.ArgumentNotNull("cluster");
VerifyApplicationHistorySupport(cluster);
this.Cluster = cluster;
this.HttpCredentials = GetClusterHttpCredentials(this.Cluster);
this.DefaultStorageCredentials = GetClusterStorageCredentials(this.Cluster);
this.timeout = timeout;
this.customUserAgent = customUserAgent;
// Versioning of application history client to be handled here.
this.applicationHistoryClient = HadoopApplicationHistoryClientFactory.Connect(this.HttpCredentials);
}
private static void AssertConfigOptionsAllowed(ClusterDetails cluster, IEnumerable<Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2013.Property> propertyList, string settingsName)
{
var nonOverridableProperty = propertyList.FirstOrDefault(prop => nonOverridableConfigurationProperties.Contains(prop.Name));
if (nonOverridableProperty != null)
{
// 'Specified Custom Settings are not valid. Error report: The following properties cannot be overridden for component: Core
// fs.default.name
cluster.Error = new ClusterErrorStatus(
(int)HttpStatusCode.BadRequest,
string.Format(
CultureInfo.InvariantCulture,
"Specified Custom Settings are not valid. Error report: The following properties cannot be overridden for component: {0} {1} {2}",
settingsName,
Environment.NewLine,
nonOverridableProperty.Name),
"Create");
}
}
private static AzureHDInsightClusterConfiguration GetAzureHDInsightClusterConfigurationV3(
ClusterDetails cluster, Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters clusterCreateDetails)
{
var azureClusterConfig = new AzureHDInsightClusterConfiguration();
var yarn = clusterCreateDetails.Components.OfType<YarnComponent>().Single();
var mapreduce = yarn.Applications.OfType<MapReduceApplication>().Single();
var hive = clusterCreateDetails.Components.OfType<HiveComponent>().Single();
var oozie = clusterCreateDetails.Components.OfType<OozieComponent>().Single();
var hdfs = clusterCreateDetails.Components.OfType<HdfsComponent>().Single();
var hadoopCore = clusterCreateDetails.Components.OfType<HadoopCoreComponent>().Single();
if (hadoopCore.CoreSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, hadoopCore.CoreSiteXmlProperties, "Core");
azureClusterConfig.Core.AddRange(
hadoopCore.CoreSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (hive.HiveSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, hive.HiveSiteXmlProperties, "Hive");
azureClusterConfig.Hive.AddRange(hive.HiveSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (hdfs.HdfsSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, hdfs.HdfsSiteXmlProperties, "Hdfs");
azureClusterConfig.Hdfs.AddRange(hdfs.HdfsSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (mapreduce.MapRedSiteXmlProperties.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, mapreduce.MapRedSiteXmlProperties, "MapReduce");
azureClusterConfig.MapReduce.AddRange(
mapreduce.MapRedSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (oozie.Configuration.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, oozie.Configuration, "Oozie");
azureClusterConfig.Oozie.AddRange(oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (yarn.Configuration.Any())
{
AssertConfigOptionsAllowedMay2014(cluster, yarn.Configuration, "Yarn");
azureClusterConfig.Yarn.AddRange(oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
return azureClusterConfig;
}
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters)
{
this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location);
LastCreateRequest = clusterCreateParameters;
var clusterDetails = new ClusterDetails();
clusterDetails.Name = clusterCreateParameters.Name;
clusterDetails.HttpPassword = clusterCreateParameters.Password;
clusterDetails.HttpUserName = clusterCreateParameters.UserName;
clusterDetails.Version = clusterCreateParameters.Version;
clusterDetails.Location = clusterCreateParameters.Location;
clusterDetails.State = ClusterState.Running;
clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts;
clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration(
clusterCreateParameters.DefaultStorageAccountName,
clusterCreateParameters.DefaultStorageAccountKey,
clusterCreateParameters.DefaultStorageContainer);
Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails });
return TaskEx2.FromResult(clusterDetails);
}
internal static AzureHDInsightJob RunJobInPowershell(
IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster)
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
IEnumerable<AzureHDInsightJob> jobCreationCmdletResults = results.Results.ToEnumerable<AzureHDInsightJob>();
AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();
Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
return jobCreationResults;
}
public static string ClusterDetailsAsString(ClusterDetails c)
{
var sb = new StringBuilder();
sb.AppendLine("Name: ".PadRight(30) + c.Name);
sb.AppendLine("Location: ".PadRight(30) + c.Location);
sb.AppendLine("ClusterType: ".PadRight(30) + c.ClusterType);
sb.AppendLine("Version: ".PadRight(30) + c.Version);
sb.AppendLine("ClusterSizeInNodes: ".PadRight(30) + c.ClusterSizeInNodes);
sb.AppendLine("State: ".PadRight(30) + c.State);
sb.AppendLine("CreatedDate: ".PadRight(30) + c.CreatedDate);
sb.AppendLine("DefaultStorageAccount: ".PadRight(30) + (c.DefaultStorageAccount == null ? "null" : c.DefaultStorageAccount.Name));
sb.AppendLine("ConnectionUrl: ".PadRight(30) + c.ConnectionUrl);
sb.AppendLine("HttpUserName: "******"HttpPassword: "******"VirtualNetworkId: ".PadRight(30) + c.VirtualNetworkId);
}
if (!String.IsNullOrWhiteSpace(c.SubnetName))
{
sb.AppendLine("SubnetName: ".PadRight(30) + c.SubnetName);
}
return sb.ToString();
}
internal static ClusterDetails GetHttpAccessDisabledCluster()
{
if (httpDisabledCluster == null)
{
lock (lockObject)
{
if (httpDisabledCluster == null)
{
var credentials = IntegrationTestBase.GetValidCredentials();
var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate));
var randomCluster = GetRandomCluster();
randomCluster.UserName = IntegrationTestBase.TestCredentials.AzureUserName;
randomCluster.Password = IntegrationTestBase.TestCredentials.AzurePassword;
var cluster = client.CreateCluster(randomCluster);
client.DisableHttp(cluster.Name, cluster.Location);
cluster = client.GetCluster(cluster.Name);
httpDisabledCluster = cluster;
}
}
}
return httpDisabledCluster;
// Creates the client
//var credentials = IntegrationTestBase.GetValidCredentials();
//var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate));
//client.DisableHttp(IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName, IntegrationTestBase.TestCredentials.Environments[0].Location);
//var testCluster = client.GetCluster(IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName);
//return testCluster;
}
/// <summary>
/// Connects to HDInsight cluster.
/// </summary>
/// <param name="certificate">The certificate.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="clusterName">Name of the cluster.</param>
/// <param name="storageAccountName">Name of the storage account.</param>
/// <param name="storageAccountKey">The storage account key.</param>
public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey)
{
// Obtain the certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast<X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0);
if (cert == null)
{
AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!");
}
// Connect to the cluster using the certificate and the subscription
try
{
this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert));
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e);
}
this.cluster = this.client.GetCluster(clusterName);
if (this.cluster == null)
{
AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName);
}
// Create a job client
this.job = JobSubmissionClientFactory.Connect(
new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName));
// Create an Azure storage client
// We will use this client to upload files to Azure storage account
// which is used by HDInsight cluster.
var storageAccount = CloudStorageAccount.Parse(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
var blobClient = storageAccount.CreateCloudBlobClient();
this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container);
}
private static AzureHDInsightJob TestJobStop(ClusterDetails testCluster, string jobId)
{
IStopAzureHDInsightJobCommand stopJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateStopJob();
stopJobCommand.Cluster = testCluster.ConnectionUrl;
stopJobCommand.Credential = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword);
stopJobCommand.JobId = jobId;
stopJobCommand.EndProcessing();
if (stopJobCommand.Output.Count != 0)
{
AzureHDInsightJob jobCancellationResults = stopJobCommand.Output.ElementAt(0);
Assert.IsNotNull(jobCancellationResults.JobId, "Should get a non-null jobDetails id");
Assert.IsNotNull(jobCancellationResults.StatusDirectory, "StatusDirectory should be set on jobDetails");
return jobCancellationResults;
}
return null;
}
private static AzureHDInsightJob TestJobStart(AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails testCluster)
{
IStartAzureHDInsightJobCommand startJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateStartJob();
startJobCommand.Cluster = testCluster.ConnectionUrl;
startJobCommand.Credential = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword);
startJobCommand.JobDefinition = mapReduceJobDefinition;
startJobCommand.EndProcessing();
AzureHDInsightJob jobCreationResults = startJobCommand.Output.ElementAt(0);
Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
Assert.IsNotNull(jobCreationResults.StatusDirectory, "StatusDirectory should be set on jobDetails");
return jobCreationResults;
}
internal static void ValidateClusterConfiguration(ClusterDetails testCluster, ClusterCreateParameters cluster)
{
var remoteCreds = new BasicAuthCredential()
{
Server = GatewayUriResolver.GetGatewayUri(new Uri(testCluster.ConnectionUrl).Host),
UserName = testCluster.HttpUserName,
Password = testCluster.HttpPassword
};
var configurationAccessor =
ServiceLocator.Instance.Locate<IAzureHDInsightClusterConfigurationAccessorFactory>().Create(remoteCreds);
var coreConfiguration = configurationAccessor.GetCoreServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.CoreConfiguration, coreConfiguration);
var hdfsConfiguration = configurationAccessor.GetHdfsServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HdfsConfiguration, hdfsConfiguration);
var mapReduceConfiguration = configurationAccessor.GetMapReduceServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.MapReduceConfiguration.ConfigurationCollection, mapReduceConfiguration);
var hiveConfiguration = configurationAccessor.GetHiveServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.HiveConfiguration.ConfigurationCollection, hiveConfiguration);
var oozieConfiguration = configurationAccessor.GetOozieServiceConfiguration().WaitForResult();
ValidateConfiguration(cluster.OozieConfiguration.ConfigurationCollection, oozieConfiguration);
}
public void ICanContinuePollingWhenClusterStateIsValid()
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
using (var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false))
{
var test = new ClusterDetails("TestCluster", ClusterState.ReadyForDeployment.ToString());
var result = client.PollSignal(test, ClusterState.Operational, ClusterState.Running);
Assert.AreEqual(result, IHDInsightManagementPocoClientExtensions.PollResult.Continue);
}
}
public void ICanStopPollingWhenErrorIsDetected()
{
IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials();
using (var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false))
{
var test = new ClusterDetails("TestCluster", ClusterState.Unknown.ToString());
test.Error = new ClusterErrorStatus(400, "Hit an error", "Create");
var result = client.PollSignal(test, ClusterState.Operational, ClusterState.Running);
Assert.AreEqual(result, IHDInsightManagementPocoClientExtensions.PollResult.Stop);
test.State = ClusterState.ClusterStorageProvisioned;
result = client.PollSignal(test, ClusterState.Operational, ClusterState.Running);
Assert.AreEqual(result, IHDInsightManagementPocoClientExtensions.PollResult.Stop);
}
}
/// <summary>
/// Initializes a new instance of the ClusterProvisioningStatusEventArgs class.
/// </summary>
/// <param name="clusterDetails">Details of the cluster being provisioned.</param>
/// <param name="clusterState">Current state of the cluster.</param>
public ClusterProvisioningStatusEventArgs(ClusterDetails clusterDetails, ClusterState clusterState)
{
this.Cluster = clusterDetails;
this.State = clusterState;
}
/// <summary>
/// Initializes a new instance of the HDInsightApplicationHistoryClient class.
/// </summary>
/// <param name="cluster">
/// Details about the cluster against which the client should be created.
/// </param>
/// <param name="timeout">
/// The timeout value to use for operations performed by this client.
/// </param>
internal HDInsightApplicationHistoryClient(ClusterDetails cluster, TimeSpan timeout)
: this(cluster, timeout, string.Empty)
{
}
private static WindowsAzureStorageAccountCredentials GetClusterStorageCredentials(ClusterDetails cluster)
{
if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Name))
{
throw new InvalidOperationException("Could not obtain default storage account for the cluster.");
}
if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Key))
{
throw new InvalidOperationException("Could not obtain credentials for default storage account of the cluster.");
}
if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Container))
{
throw new InvalidOperationException("Could not obtain information about default container of the cluster");
}
return(new WindowsAzureStorageAccountCredentials()
{
Name = cluster.DefaultStorageAccount.Name,
Key = cluster.DefaultStorageAccount.Key,
ContainerName = cluster.DefaultStorageAccount.Container
});
}
internal static ClusterDetails GetHttpAccessEnabledCluster(bool reEnable)
{
// creates the client
if (httpEnabledCluster == null)
{
lock (lockObject)
{
if (httpEnabledCluster == null)
{
var credentials = IntegrationTestBase.GetValidCredentials();
var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate));
var randomCluster = GetRandomCluster();
randomCluster.UserName = IntegrationTestBase.TestCredentials.AzureUserName;
randomCluster.Password = IntegrationTestBase.TestCredentials.AzurePassword;
httpEnabledCluster = client.CreateCluster(randomCluster);
}
}
}
return httpEnabledCluster;
//var clusters = client.ListClusters();
//if (reEnable)
//{
// var clusterWithHttpAccessDisabled = clusters.FirstOrDefault(cluster => string.IsNullOrEmpty(cluster.HttpUserName));
// if (clusterWithHttpAccessDisabled == null)
// {
// // if http access is enabled, disable it on the last cluster in the list.
// clusterWithHttpAccessDisabled = clusters.Last();
// client.DisableHttp(clusterWithHttpAccessDisabled.Name, clusterWithHttpAccessDisabled.Location);
// }
// client.EnableHttp(
// clusterWithHttpAccessDisabled.Name,
// clusterWithHttpAccessDisabled.Location,
// IntegrationTestBase.TestCredentials.AzureUserName,
// IntegrationTestBase.TestCredentials.AzurePassword);
// testCluster = client.GetCluster(clusterWithHttpAccessDisabled.Name);
//}
//else
//{
// testCluster = clusters.FirstOrDefault(cluster => !string.IsNullOrEmpty(cluster.HttpUserName));
// if (testCluster == null)
// {
// testCluster = GetHttpAccessEnabledCluster(true);
// }
//}
//testCluster.ConnectionUrl = GatewayUriResolver.GetGatewayUri(testCluster.ConnectionUrl).AbsoluteUri;
//Debug.WriteLine(string.Format("Searched for Http Enabled Cluster, Returning '{0}'", testCluster.Name));
//return testCluster;
}
/// <summary>
/// Initializes a new instance of the HDInsightApplicationHistoryClient class.
/// </summary>
/// <param name="cluster">
/// Details about the cluster against which the client should be created.
/// </param>
internal HDInsightApplicationHistoryClient(ClusterDetails cluster)
: this(cluster, TimeSpan.FromMinutes(5))
{
}
/// <summary>
/// Initializes a new instance of the HDInsightApplicationHistoryClient class.
/// </summary>
/// <param name="cluster">
/// Details about the cluster against which the client should be created.
/// </param>
/// <param name="timeout">
/// The timeout value to use for operations performed by this client.
/// </param>
internal HDInsightApplicationHistoryClient(ClusterDetails cluster, TimeSpan timeout)
: this(cluster, timeout, string.Empty)
{
}
/// <summary>
/// Used to handle the notification events during waiting.
/// </summary>
/// <param name="cluster">
/// The cluster in its current state.
/// </param>
public void HandleClusterWaitNotifyEvent(ClusterDetails cluster)
{
if (cluster.IsNotNull())
{
this.currentDetails = cluster;
this.RaiseClusterProvisioningEvent(this, new ClusterProvisioningStatusEventArgs(cluster, cluster.State));
}
}
public static ClusterDetails CreateHBaseClusterIfNotExists()
{
HBaseCluster = CreateClusterIfNotExists(ClusterType.HBase);
return HBaseCluster;
}
private static BasicAuthCredential GetClusterHttpCredentials(ClusterDetails cluster)
{
if (string.IsNullOrEmpty(cluster.ConnectionUrl))
{
throw new InvalidOperationException("Unable to connect to cluster as connection url is missing or empty.");
}
if (string.IsNullOrEmpty(cluster.HttpUserName) || string.IsNullOrEmpty(cluster.HttpPassword))
{
throw new InvalidOperationException("Unable to connect to cluster as cluster username and/or password are missing or empty.");
}
BasicAuthCredential clusterCreds = new BasicAuthCredential()
{
Server = new Uri(cluster.ConnectionUrl),
UserName = cluster.HttpUserName,
Password = cluster.HttpPassword
};
return clusterCreds;
}
internal static AzureHDInsightJob RunJobInPowershell(
IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster, bool debug, string expectedLogMessage)
{
IPipelineResult result = null;
if (debug)
{
var logWriter = new PowershellLogWriter();
BufferingLogWriterFactory.Instance = logWriter;
result =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.WithParameter(CmdletConstants.Debug, null)
.Invoke();
Assert.IsTrue(logWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
BufferingLogWriterFactory.Reset();
}
else
{
result =
runspace.NewPipeline()
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
.Invoke();
}
Assert.AreEqual(1, result.Results.Count);
IEnumerable<AzureHDInsightJob> jobCreationCmdletResults = result.Results.ToEnumerable<AzureHDInsightJob>();
AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();
Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
return jobCreationResults;
}
private static WindowsAzureStorageAccountCredentials GetClusterStorageCredentials(ClusterDetails cluster)
{
if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Name))
{
throw new InvalidOperationException("Could not obtain default storage account for the cluster.");
}
if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Key))
{
throw new InvalidOperationException("Could not obtain credentials for default storage account of the cluster.");
}
if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Container))
{
throw new InvalidOperationException("Could not obtain information about default container of the cluster");
}
return new WindowsAzureStorageAccountCredentials()
{
Name = cluster.DefaultStorageAccount.Name,
Key = cluster.DefaultStorageAccount.Key,
ContainerName = cluster.DefaultStorageAccount.Container
};
}
public async Task<IHttpResponseMessageAbstraction> CreateContainer(string dnsName, string location, string clusterPayload, int schemaVersion = 2)
{
this.LogMessage("Creating cluster '{0}' in location {1}", dnsName, location);
this.ValidateConnection();
var registrationClient = ServiceLocator.Instance.Locate<ISubscriptionRegistrationClientFactory>().Create(this.credentials, this.context, false);
if (!await registrationClient.ValidateSubscriptionLocation(location))
{
var resolver = ServiceLocator.Instance.Locate<ICloudServiceNameResolver>();
string regionCloudServicename = resolver.GetCloudServiceName(
this.credentials.SubscriptionId, this.credentials.DeploymentNamespace, location);
throw new HttpLayerException(
HttpStatusCode.NotFound, string.Format("The cloud service with name {0} was not found.", regionCloudServicename));
}
lock (Clusters)
{
var existingCluster = Clusters.FirstOrDefault(c => c.Cluster.Name == dnsName && c.Cluster.Location == location);
if (existingCluster != null)
throw new HttpLayerException(HttpStatusCode.BadRequest, "<!DOCTYPE html><html>" + HDInsightClient.ClusterAlreadyExistsError + "</html>");
HDInsight.ClusterCreateParametersV2 cluster = null;
AzureHDInsightClusterConfiguration azureClusterConfig = null;
ClusterDetails createCluster = null;
if (schemaVersion == 2)
{
var resource = ServerSerializer.DeserializeClusterCreateRequestIntoResource(clusterPayload);
if (resource.SchemaVersion != "2.0")
{
throw new HttpLayerException(HttpStatusCode.BadRequest, "SchemaVersion needs to be at 2.0");
}
cluster = ServerSerializer.DeserializeClusterCreateRequest(clusterPayload);
var storageAccounts = GetStorageAccounts(cluster).ToList();
createCluster = new ClusterDetails(cluster.Name, HDInsight.ClusterState.ReadyForDeployment.ToString())
{
ConnectionUrl = string.Format(@"https://{0}.azurehdinsight.net", cluster.Name),
Location = cluster.Location,
Error = this.ValidateClusterCreation(cluster),
HttpUserName = cluster.UserName,
HttpPassword = cluster.Password,
Version = this.GetVersion(cluster.Version),
ClusterSizeInNodes = cluster.ClusterSizeInNodes,
DefaultStorageAccount = storageAccounts.FirstOrDefault(),
AdditionalStorageAccounts = storageAccounts.Skip(1),
ClusterType = cluster.ClusterType
};
var clusterCreateDetails = ServerSerializer.DeserializeClusterCreateRequestToInternal(clusterPayload);
azureClusterConfig = GetAzureHDInsightClusterConfiguration(createCluster, clusterCreateDetails);
}
else if (schemaVersion == 3)
{
var resource = ServerSerializer.DeserializeClusterCreateRequestIntoResource(clusterPayload);
if (resource.SchemaVersion != "3.0")
{
throw new HttpLayerException(HttpStatusCode.BadRequest, "SchemaVersion needs to be at 3.0");
}
cluster = ServerSerializer.DeserializeClusterCreateRequestV3(clusterPayload);
var storageAccounts = GetStorageAccounts(cluster).ToList();
createCluster = new ClusterDetails(cluster.Name, HDInsight.ClusterState.ReadyForDeployment.ToString())
{
ConnectionUrl = string.Format(@"https://{0}.azurehdinsight.net", cluster.Name),
Location = cluster.Location,
Error = this.ValidateClusterCreation(cluster),
HttpUserName = cluster.UserName,
HttpPassword = cluster.Password,
Version = this.GetVersion(cluster.Version),
ClusterSizeInNodes = cluster.ClusterSizeInNodes,
DefaultStorageAccount = storageAccounts.FirstOrDefault(),
AdditionalStorageAccounts = storageAccounts.Skip(1),
ClusterType = cluster.ClusterType
};
var clusterCreateDetails = ServerSerializer.DeserializeClusterCreateRequestToInternalV3(clusterPayload);
azureClusterConfig = GetAzureHDInsightClusterConfigurationV3(createCluster, clusterCreateDetails);
}
else
{
throw new HttpLayerException(HttpStatusCode.BadRequest, "Invalid SchemaVersion");
}
var simCluster = new SimulatorClusterContainer() { Cluster = createCluster, Configuration = azureClusterConfig };
Clusters.Add(simCluster);
if (createCluster.Name.Contains("NetworkExceptionButCreateSucceeds"))
{
throw new HttpRequestException("An error occurred while sending the request.");
}
if (createCluster.Name.Contains("HttpErrorButCreateSucceeds"))
{
return new HttpResponseMessageAbstraction(HttpStatusCode.BadGateway, null, "BadGateway");
}
}
return new HttpResponseMessageAbstraction(HttpStatusCode.Accepted, null, string.Empty);
}
private static void VerifyApplicationHistorySupport(ClusterDetails cluster)
{
// Verify cluster type
if (cluster.ClusterType != ClusterType.Hadoop)
{
throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, @"Cannot get application history for clusters of type '{0}'. This functionality is supported for Hadoop clusters only", cluster.ClusterType));
}
// Verify cluster version
string minSupportedVersion = "3.1.1.374";
Version clusterVersion = new Version(cluster.Version);
if (clusterVersion.CompareTo(new Version(minSupportedVersion)) < 0)
{
throw new NotSupportedException(
string.Format(CultureInfo.InvariantCulture,
"Cannot get application history for cluster with version '{0}'. This functionality is supported for clusters with versions {1} and above",
cluster.Version,
minSupportedVersion));
}
}
private static AzureHDInsightClusterConfiguration GetAzureHDInsightClusterConfiguration(
ClusterDetails cluster, ClusterContainer clusterCreateDetails)
{
var azureClusterConfig = new AzureHDInsightClusterConfiguration();
if (clusterCreateDetails.Settings.Core != null && clusterCreateDetails.Settings.Core.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Core.Configuration, "Core");
azureClusterConfig.Core.AddRange(
clusterCreateDetails.Settings.Core.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.Hive != null && clusterCreateDetails.Settings.Hive.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Hive.Configuration, "Hive");
azureClusterConfig.Hive.AddRange(
clusterCreateDetails.Settings.Hive.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.Hdfs != null && clusterCreateDetails.Settings.Hdfs.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Hdfs.Configuration, "Hdfs");
azureClusterConfig.Hdfs.AddRange(
clusterCreateDetails.Settings.Hdfs.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.MapReduce != null && clusterCreateDetails.Settings.MapReduce.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.MapReduce.Configuration, "MapReduce");
azureClusterConfig.MapReduce.AddRange(
clusterCreateDetails.Settings.MapReduce.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
if (clusterCreateDetails.Settings.Oozie != null && clusterCreateDetails.Settings.Oozie.Configuration != null)
{
AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Oozie.Configuration, "Oozie");
azureClusterConfig.Oozie.AddRange(
clusterCreateDetails.Settings.Oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value)));
}
return azureClusterConfig;
}
/// <summary>
/// Initializes a new instance of the HDInsightApplicationHistoryClient class.
/// </summary>
/// <param name="cluster">
/// Details about the cluster against which the client should be created.
/// </param>
internal HDInsightApplicationHistoryClient(ClusterDetails cluster)
: this(cluster, TimeSpan.FromMinutes(5))
{
}
internal static AzureHDInsightJob GetJobWithID(IRunspace runspace, string jobId, ClusterDetails cluster)
{
IPipelineResult getJobDetailResults =
runspace.NewPipeline()
.AddCommand(CmdletConstants.GetAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.Id, jobId)
.Invoke();
return getJobDetailResults.Results.ToEnumerable<AzureHDInsightJob>().FirstOrDefault();
}
/// <summary>
/// Initializes a new instance of the ClusterProvisioningStatusEventArgs class.
/// </summary>
/// <param name="clusterDetails">Details of the cluster being provisioned.</param>
/// <param name="clusterState">Current state of the cluster.</param>
public ClusterProvisioningStatusEventArgs(ClusterDetails clusterDetails, ClusterState clusterState)
{
this.Cluster = clusterDetails;
this.State = clusterState;
}
