public HDInsightClusterCreateException(ClusterDetails clusterDetails) : this(clusterDetails != null ? string.Format(CultureInfo.InvariantCulture, "Unable to complete the cluster create operation. Operation failed with code '{0}'. Cluster left behind state: '{1}'. Message: '{2}'.", clusterDetails.Error.HttpCode, clusterDetails.StateString ?? "NULL", clusterDetails.Error.Message ?? "NULL") : DefaultExceptionMessage, null) { this.hdiClusterDetails = clusterDetails; }
/// <summary> /// Initializes a new instance of the HDInsightApplicationHistoryClient class. /// </summary> /// <param name="cluster"> /// Details about the cluster against which the client should be created. /// </param> /// <param name="timeout"> /// The timeout value to use for operations performed by this client. /// </param> /// <param name="customUserAgent"> /// A tag to help identify the client/user. /// </param> internal HDInsightApplicationHistoryClient(ClusterDetails cluster, TimeSpan timeout, string customUserAgent) { cluster.ArgumentNotNull("cluster"); VerifyApplicationHistorySupport(cluster); this.Cluster = cluster; this.HttpCredentials = GetClusterHttpCredentials(this.Cluster); this.DefaultStorageCredentials = GetClusterStorageCredentials(this.Cluster); this.timeout = timeout; this.customUserAgent = customUserAgent; // Versioning of application history client to be handled here. this.applicationHistoryClient = HadoopApplicationHistoryClientFactory.Connect(this.HttpCredentials); }
private static void AssertConfigOptionsAllowed(ClusterDetails cluster, IEnumerable<Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2013.Property> propertyList, string settingsName) { var nonOverridableProperty = propertyList.FirstOrDefault(prop => nonOverridableConfigurationProperties.Contains(prop.Name)); if (nonOverridableProperty != null) { // 'Specified Custom Settings are not valid. Error report: The following properties cannot be overridden for component: Core // fs.default.name cluster.Error = new ClusterErrorStatus( (int)HttpStatusCode.BadRequest, string.Format( CultureInfo.InvariantCulture, "Specified Custom Settings are not valid. Error report: The following properties cannot be overridden for component: {0} {1} {2}", settingsName, Environment.NewLine, nonOverridableProperty.Name), "Create"); } }
private static AzureHDInsightClusterConfiguration GetAzureHDInsightClusterConfigurationV3( ClusterDetails cluster, Microsoft.WindowsAzure.Management.HDInsight.Contracts.May2014.ClusterCreateParameters clusterCreateDetails) { var azureClusterConfig = new AzureHDInsightClusterConfiguration(); var yarn = clusterCreateDetails.Components.OfType<YarnComponent>().Single(); var mapreduce = yarn.Applications.OfType<MapReduceApplication>().Single(); var hive = clusterCreateDetails.Components.OfType<HiveComponent>().Single(); var oozie = clusterCreateDetails.Components.OfType<OozieComponent>().Single(); var hdfs = clusterCreateDetails.Components.OfType<HdfsComponent>().Single(); var hadoopCore = clusterCreateDetails.Components.OfType<HadoopCoreComponent>().Single(); if (hadoopCore.CoreSiteXmlProperties.Any()) { AssertConfigOptionsAllowedMay2014(cluster, hadoopCore.CoreSiteXmlProperties, "Core"); azureClusterConfig.Core.AddRange( hadoopCore.CoreSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (hive.HiveSiteXmlProperties.Any()) { AssertConfigOptionsAllowedMay2014(cluster, hive.HiveSiteXmlProperties, "Hive"); azureClusterConfig.Hive.AddRange(hive.HiveSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (hdfs.HdfsSiteXmlProperties.Any()) { AssertConfigOptionsAllowedMay2014(cluster, hdfs.HdfsSiteXmlProperties, "Hdfs"); azureClusterConfig.Hdfs.AddRange(hdfs.HdfsSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (mapreduce.MapRedSiteXmlProperties.Any()) { AssertConfigOptionsAllowedMay2014(cluster, mapreduce.MapRedSiteXmlProperties, "MapReduce"); azureClusterConfig.MapReduce.AddRange( mapreduce.MapRedSiteXmlProperties.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (oozie.Configuration.Any()) { AssertConfigOptionsAllowedMay2014(cluster, oozie.Configuration, "Oozie"); azureClusterConfig.Oozie.AddRange(oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (yarn.Configuration.Any()) { AssertConfigOptionsAllowedMay2014(cluster, yarn.Configuration, "Yarn"); azureClusterConfig.Yarn.AddRange(oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } return azureClusterConfig; }
public Task<ClusterDetails> CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters) { this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location); LastCreateRequest = clusterCreateParameters; var clusterDetails = new ClusterDetails(); clusterDetails.Name = clusterCreateParameters.Name; clusterDetails.HttpPassword = clusterCreateParameters.Password; clusterDetails.HttpUserName = clusterCreateParameters.UserName; clusterDetails.Version = clusterCreateParameters.Version; clusterDetails.Location = clusterCreateParameters.Location; clusterDetails.State = ClusterState.Running; clusterDetails.AdditionalStorageAccounts = clusterCreateParameters.AdditionalStorageAccounts; clusterDetails.DefaultStorageAccount = new WabStorageAccountConfiguration( clusterCreateParameters.DefaultStorageAccountName, clusterCreateParameters.DefaultStorageAccountKey, clusterCreateParameters.DefaultStorageContainer); Clusters.Add(new SimulatorClusterContainer { Cluster = clusterDetails }); return TaskEx2.FromResult(clusterDetails); }
internal static AzureHDInsightJob RunJobInPowershell( IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.StartAzureHDInsightJob) .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl) .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword)) .WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition) .Invoke(); Assert.AreEqual(1, results.Results.Count); IEnumerable<AzureHDInsightJob> jobCreationCmdletResults = results.Results.ToEnumerable<AzureHDInsightJob>(); AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First(); Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id"); return jobCreationResults; }
public static string ClusterDetailsAsString(ClusterDetails c) { var sb = new StringBuilder(); sb.AppendLine("Name: ".PadRight(30) + c.Name); sb.AppendLine("Location: ".PadRight(30) + c.Location); sb.AppendLine("ClusterType: ".PadRight(30) + c.ClusterType); sb.AppendLine("Version: ".PadRight(30) + c.Version); sb.AppendLine("ClusterSizeInNodes: ".PadRight(30) + c.ClusterSizeInNodes); sb.AppendLine("State: ".PadRight(30) + c.State); sb.AppendLine("CreatedDate: ".PadRight(30) + c.CreatedDate); sb.AppendLine("DefaultStorageAccount: ".PadRight(30) + (c.DefaultStorageAccount == null ? "null" : c.DefaultStorageAccount.Name)); sb.AppendLine("ConnectionUrl: ".PadRight(30) + c.ConnectionUrl); sb.AppendLine("HttpUserName: "******"HttpPassword: "******"VirtualNetworkId: ".PadRight(30) + c.VirtualNetworkId); } if (!String.IsNullOrWhiteSpace(c.SubnetName)) { sb.AppendLine("SubnetName: ".PadRight(30) + c.SubnetName); } return sb.ToString(); }
internal static ClusterDetails GetHttpAccessDisabledCluster() { if (httpDisabledCluster == null) { lock (lockObject) { if (httpDisabledCluster == null) { var credentials = IntegrationTestBase.GetValidCredentials(); var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate)); var randomCluster = GetRandomCluster(); randomCluster.UserName = IntegrationTestBase.TestCredentials.AzureUserName; randomCluster.Password = IntegrationTestBase.TestCredentials.AzurePassword; var cluster = client.CreateCluster(randomCluster); client.DisableHttp(cluster.Name, cluster.Location); cluster = client.GetCluster(cluster.Name); httpDisabledCluster = cluster; } } } return httpDisabledCluster; // Creates the client //var credentials = IntegrationTestBase.GetValidCredentials(); //var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate)); //client.DisableHttp(IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName, IntegrationTestBase.TestCredentials.Environments[0].Location); //var testCluster = client.GetCluster(IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName); //return testCluster; }
/// <summary> /// Connects to HDInsight cluster. /// </summary> /// <param name="certificate">The certificate.</param> /// <param name="subscription">The subscription.</param> /// <param name="clusterName">Name of the cluster.</param> /// <param name="storageAccountName">Name of the storage account.</param> /// <param name="storageAccountKey">The storage account key.</param> public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey) { // Obtain the certificate var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast<X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0); if (cert == null) { AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!"); } // Connect to the cluster using the certificate and the subscription try { this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert)); } catch (Exception e) { AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e); } this.cluster = this.client.GetCluster(clusterName); if (this.cluster == null) { AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName); } // Create a job client this.job = JobSubmissionClientFactory.Connect( new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName)); // Create an Azure storage client // We will use this client to upload files to Azure storage account // which is used by HDInsight cluster. var storageAccount = CloudStorageAccount.Parse( "DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey); var blobClient = storageAccount.CreateCloudBlobClient(); this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container); }
private static AzureHDInsightJob TestJobStop(ClusterDetails testCluster, string jobId) { IStopAzureHDInsightJobCommand stopJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateStopJob(); stopJobCommand.Cluster = testCluster.ConnectionUrl; stopJobCommand.Credential = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword); stopJobCommand.JobId = jobId; stopJobCommand.EndProcessing(); if (stopJobCommand.Output.Count != 0) { AzureHDInsightJob jobCancellationResults = stopJobCommand.Output.ElementAt(0); Assert.IsNotNull(jobCancellationResults.JobId, "Should get a non-null jobDetails id"); Assert.IsNotNull(jobCancellationResults.StatusDirectory, "StatusDirectory should be set on jobDetails"); return jobCancellationResults; } return null; }
private static AzureHDInsightJob TestJobStart(AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails testCluster) { IStartAzureHDInsightJobCommand startJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateStartJob(); startJobCommand.Cluster = testCluster.ConnectionUrl; startJobCommand.Credential = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword); startJobCommand.JobDefinition = mapReduceJobDefinition; startJobCommand.EndProcessing(); AzureHDInsightJob jobCreationResults = startJobCommand.Output.ElementAt(0); Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id"); Assert.IsNotNull(jobCreationResults.StatusDirectory, "StatusDirectory should be set on jobDetails"); return jobCreationResults; }
internal static void ValidateClusterConfiguration(ClusterDetails testCluster, ClusterCreateParameters cluster) { var remoteCreds = new BasicAuthCredential() { Server = GatewayUriResolver.GetGatewayUri(new Uri(testCluster.ConnectionUrl).Host), UserName = testCluster.HttpUserName, Password = testCluster.HttpPassword }; var configurationAccessor = ServiceLocator.Instance.Locate<IAzureHDInsightClusterConfigurationAccessorFactory>().Create(remoteCreds); var coreConfiguration = configurationAccessor.GetCoreServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.CoreConfiguration, coreConfiguration); var hdfsConfiguration = configurationAccessor.GetHdfsServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.HdfsConfiguration, hdfsConfiguration); var mapReduceConfiguration = configurationAccessor.GetMapReduceServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.MapReduceConfiguration.ConfigurationCollection, mapReduceConfiguration); var hiveConfiguration = configurationAccessor.GetHiveServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.HiveConfiguration.ConfigurationCollection, hiveConfiguration); var oozieConfiguration = configurationAccessor.GetOozieServiceConfiguration().WaitForResult(); ValidateConfiguration(cluster.OozieConfiguration.ConfigurationCollection, oozieConfiguration); }
public void ICanContinuePollingWhenClusterStateIsValid() { IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials(); using (var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false)) { var test = new ClusterDetails("TestCluster", ClusterState.ReadyForDeployment.ToString()); var result = client.PollSignal(test, ClusterState.Operational, ClusterState.Running); Assert.AreEqual(result, IHDInsightManagementPocoClientExtensions.PollResult.Continue); } }
public void ICanStopPollingWhenErrorIsDetected() { IHDInsightCertificateCredential credentials = IntegrationTestBase.GetValidCredentials(); using (var client = ServiceLocator.Instance.Locate<IHDInsightManagementPocoClientFactory>().Create(credentials, GetAbstractionContext(), false)) { var test = new ClusterDetails("TestCluster", ClusterState.Unknown.ToString()); test.Error = new ClusterErrorStatus(400, "Hit an error", "Create"); var result = client.PollSignal(test, ClusterState.Operational, ClusterState.Running); Assert.AreEqual(result, IHDInsightManagementPocoClientExtensions.PollResult.Stop); test.State = ClusterState.ClusterStorageProvisioned; result = client.PollSignal(test, ClusterState.Operational, ClusterState.Running); Assert.AreEqual(result, IHDInsightManagementPocoClientExtensions.PollResult.Stop); } }
/// <summary> /// Initializes a new instance of the ClusterProvisioningStatusEventArgs class. /// </summary> /// <param name="clusterDetails">Details of the cluster being provisioned.</param> /// <param name="clusterState">Current state of the cluster.</param> public ClusterProvisioningStatusEventArgs(ClusterDetails clusterDetails, ClusterState clusterState) { this.Cluster = clusterDetails; this.State = clusterState; }
/// <summary> /// Initializes a new instance of the HDInsightApplicationHistoryClient class. /// </summary> /// <param name="cluster"> /// Details about the cluster against which the client should be created. /// </param> /// <param name="timeout"> /// The timeout value to use for operations performed by this client. /// </param> internal HDInsightApplicationHistoryClient(ClusterDetails cluster, TimeSpan timeout) : this(cluster, timeout, string.Empty) { }
private static WindowsAzureStorageAccountCredentials GetClusterStorageCredentials(ClusterDetails cluster) { if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Name)) { throw new InvalidOperationException("Could not obtain default storage account for the cluster."); } if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Key)) { throw new InvalidOperationException("Could not obtain credentials for default storage account of the cluster."); } if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Container)) { throw new InvalidOperationException("Could not obtain information about default container of the cluster"); } return(new WindowsAzureStorageAccountCredentials() { Name = cluster.DefaultStorageAccount.Name, Key = cluster.DefaultStorageAccount.Key, ContainerName = cluster.DefaultStorageAccount.Container }); }
internal static ClusterDetails GetHttpAccessEnabledCluster(bool reEnable) { // creates the client if (httpEnabledCluster == null) { lock (lockObject) { if (httpEnabledCluster == null) { var credentials = IntegrationTestBase.GetValidCredentials(); var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate)); var randomCluster = GetRandomCluster(); randomCluster.UserName = IntegrationTestBase.TestCredentials.AzureUserName; randomCluster.Password = IntegrationTestBase.TestCredentials.AzurePassword; httpEnabledCluster = client.CreateCluster(randomCluster); } } } return httpEnabledCluster; //var clusters = client.ListClusters(); //if (reEnable) //{ // var clusterWithHttpAccessDisabled = clusters.FirstOrDefault(cluster => string.IsNullOrEmpty(cluster.HttpUserName)); // if (clusterWithHttpAccessDisabled == null) // { // // if http access is enabled, disable it on the last cluster in the list. // clusterWithHttpAccessDisabled = clusters.Last(); // client.DisableHttp(clusterWithHttpAccessDisabled.Name, clusterWithHttpAccessDisabled.Location); // } // client.EnableHttp( // clusterWithHttpAccessDisabled.Name, // clusterWithHttpAccessDisabled.Location, // IntegrationTestBase.TestCredentials.AzureUserName, // IntegrationTestBase.TestCredentials.AzurePassword); // testCluster = client.GetCluster(clusterWithHttpAccessDisabled.Name); //} //else //{ // testCluster = clusters.FirstOrDefault(cluster => !string.IsNullOrEmpty(cluster.HttpUserName)); // if (testCluster == null) // { // testCluster = GetHttpAccessEnabledCluster(true); // } //} //testCluster.ConnectionUrl = GatewayUriResolver.GetGatewayUri(testCluster.ConnectionUrl).AbsoluteUri; //Debug.WriteLine(string.Format("Searched for Http Enabled Cluster, Returning '{0}'", testCluster.Name)); //return testCluster; }
/// <summary> /// Initializes a new instance of the HDInsightApplicationHistoryClient class. /// </summary> /// <param name="cluster"> /// Details about the cluster against which the client should be created. /// </param> internal HDInsightApplicationHistoryClient(ClusterDetails cluster) : this(cluster, TimeSpan.FromMinutes(5)) { }
/// <summary> /// Initializes a new instance of the HDInsightApplicationHistoryClient class. /// </summary> /// <param name="cluster"> /// Details about the cluster against which the client should be created. /// </param> /// <param name="timeout"> /// The timeout value to use for operations performed by this client. /// </param> internal HDInsightApplicationHistoryClient(ClusterDetails cluster, TimeSpan timeout) : this(cluster, timeout, string.Empty) { }
/// <summary> /// Used to handle the notification events during waiting. /// </summary> /// <param name="cluster"> /// The cluster in its current state. /// </param> public void HandleClusterWaitNotifyEvent(ClusterDetails cluster) { if (cluster.IsNotNull()) { this.currentDetails = cluster; this.RaiseClusterProvisioningEvent(this, new ClusterProvisioningStatusEventArgs(cluster, cluster.State)); } }
public static ClusterDetails CreateHBaseClusterIfNotExists() { HBaseCluster = CreateClusterIfNotExists(ClusterType.HBase); return HBaseCluster; }
private static BasicAuthCredential GetClusterHttpCredentials(ClusterDetails cluster) { if (string.IsNullOrEmpty(cluster.ConnectionUrl)) { throw new InvalidOperationException("Unable to connect to cluster as connection url is missing or empty."); } if (string.IsNullOrEmpty(cluster.HttpUserName) || string.IsNullOrEmpty(cluster.HttpPassword)) { throw new InvalidOperationException("Unable to connect to cluster as cluster username and/or password are missing or empty."); } BasicAuthCredential clusterCreds = new BasicAuthCredential() { Server = new Uri(cluster.ConnectionUrl), UserName = cluster.HttpUserName, Password = cluster.HttpPassword }; return clusterCreds; }
internal static AzureHDInsightJob RunJobInPowershell( IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster, bool debug, string expectedLogMessage) { IPipelineResult result = null; if (debug) { var logWriter = new PowershellLogWriter(); BufferingLogWriterFactory.Instance = logWriter; result = runspace.NewPipeline() .AddCommand(CmdletConstants.StartAzureHDInsightJob) .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl) .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword)) .WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition) .WithParameter(CmdletConstants.Debug, null) .Invoke(); Assert.IsTrue(logWriter.Buffer.Any(message => message.Contains(expectedLogMessage))); BufferingLogWriterFactory.Reset(); } else { result = runspace.NewPipeline() .AddCommand(CmdletConstants.StartAzureHDInsightJob) .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl) .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword)) .WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition) .Invoke(); } Assert.AreEqual(1, result.Results.Count); IEnumerable<AzureHDInsightJob> jobCreationCmdletResults = result.Results.ToEnumerable<AzureHDInsightJob>(); AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First(); Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id"); return jobCreationResults; }
private static WindowsAzureStorageAccountCredentials GetClusterStorageCredentials(ClusterDetails cluster) { if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Name)) { throw new InvalidOperationException("Could not obtain default storage account for the cluster."); } if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Key)) { throw new InvalidOperationException("Could not obtain credentials for default storage account of the cluster."); } if (string.IsNullOrEmpty(cluster.DefaultStorageAccount.Container)) { throw new InvalidOperationException("Could not obtain information about default container of the cluster"); } return new WindowsAzureStorageAccountCredentials() { Name = cluster.DefaultStorageAccount.Name, Key = cluster.DefaultStorageAccount.Key, ContainerName = cluster.DefaultStorageAccount.Container }; }
public async Task<IHttpResponseMessageAbstraction> CreateContainer(string dnsName, string location, string clusterPayload, int schemaVersion = 2) { this.LogMessage("Creating cluster '{0}' in location {1}", dnsName, location); this.ValidateConnection(); var registrationClient = ServiceLocator.Instance.Locate<ISubscriptionRegistrationClientFactory>().Create(this.credentials, this.context, false); if (!await registrationClient.ValidateSubscriptionLocation(location)) { var resolver = ServiceLocator.Instance.Locate<ICloudServiceNameResolver>(); string regionCloudServicename = resolver.GetCloudServiceName( this.credentials.SubscriptionId, this.credentials.DeploymentNamespace, location); throw new HttpLayerException( HttpStatusCode.NotFound, string.Format("The cloud service with name {0} was not found.", regionCloudServicename)); } lock (Clusters) { var existingCluster = Clusters.FirstOrDefault(c => c.Cluster.Name == dnsName && c.Cluster.Location == location); if (existingCluster != null) throw new HttpLayerException(HttpStatusCode.BadRequest, "<!DOCTYPE html><html>" + HDInsightClient.ClusterAlreadyExistsError + "</html>"); HDInsight.ClusterCreateParametersV2 cluster = null; AzureHDInsightClusterConfiguration azureClusterConfig = null; ClusterDetails createCluster = null; if (schemaVersion == 2) { var resource = ServerSerializer.DeserializeClusterCreateRequestIntoResource(clusterPayload); if (resource.SchemaVersion != "2.0") { throw new HttpLayerException(HttpStatusCode.BadRequest, "SchemaVersion needs to be at 2.0"); } cluster = ServerSerializer.DeserializeClusterCreateRequest(clusterPayload); var storageAccounts = GetStorageAccounts(cluster).ToList(); createCluster = new ClusterDetails(cluster.Name, HDInsight.ClusterState.ReadyForDeployment.ToString()) { ConnectionUrl = string.Format(@"https://{0}.azurehdinsight.net", cluster.Name), Location = cluster.Location, Error = this.ValidateClusterCreation(cluster), HttpUserName = cluster.UserName, HttpPassword = cluster.Password, Version = this.GetVersion(cluster.Version), ClusterSizeInNodes = cluster.ClusterSizeInNodes, DefaultStorageAccount = storageAccounts.FirstOrDefault(), AdditionalStorageAccounts = storageAccounts.Skip(1), ClusterType = cluster.ClusterType }; var clusterCreateDetails = ServerSerializer.DeserializeClusterCreateRequestToInternal(clusterPayload); azureClusterConfig = GetAzureHDInsightClusterConfiguration(createCluster, clusterCreateDetails); } else if (schemaVersion == 3) { var resource = ServerSerializer.DeserializeClusterCreateRequestIntoResource(clusterPayload); if (resource.SchemaVersion != "3.0") { throw new HttpLayerException(HttpStatusCode.BadRequest, "SchemaVersion needs to be at 3.0"); } cluster = ServerSerializer.DeserializeClusterCreateRequestV3(clusterPayload); var storageAccounts = GetStorageAccounts(cluster).ToList(); createCluster = new ClusterDetails(cluster.Name, HDInsight.ClusterState.ReadyForDeployment.ToString()) { ConnectionUrl = string.Format(@"https://{0}.azurehdinsight.net", cluster.Name), Location = cluster.Location, Error = this.ValidateClusterCreation(cluster), HttpUserName = cluster.UserName, HttpPassword = cluster.Password, Version = this.GetVersion(cluster.Version), ClusterSizeInNodes = cluster.ClusterSizeInNodes, DefaultStorageAccount = storageAccounts.FirstOrDefault(), AdditionalStorageAccounts = storageAccounts.Skip(1), ClusterType = cluster.ClusterType }; var clusterCreateDetails = ServerSerializer.DeserializeClusterCreateRequestToInternalV3(clusterPayload); azureClusterConfig = GetAzureHDInsightClusterConfigurationV3(createCluster, clusterCreateDetails); } else { throw new HttpLayerException(HttpStatusCode.BadRequest, "Invalid SchemaVersion"); } var simCluster = new SimulatorClusterContainer() { Cluster = createCluster, Configuration = azureClusterConfig }; Clusters.Add(simCluster); if (createCluster.Name.Contains("NetworkExceptionButCreateSucceeds")) { throw new HttpRequestException("An error occurred while sending the request."); } if (createCluster.Name.Contains("HttpErrorButCreateSucceeds")) { return new HttpResponseMessageAbstraction(HttpStatusCode.BadGateway, null, "BadGateway"); } } return new HttpResponseMessageAbstraction(HttpStatusCode.Accepted, null, string.Empty); }
private static void VerifyApplicationHistorySupport(ClusterDetails cluster) { // Verify cluster type if (cluster.ClusterType != ClusterType.Hadoop) { throw new NotSupportedException(string.Format(CultureInfo.InvariantCulture, @"Cannot get application history for clusters of type '{0}'. This functionality is supported for Hadoop clusters only", cluster.ClusterType)); } // Verify cluster version string minSupportedVersion = "3.1.1.374"; Version clusterVersion = new Version(cluster.Version); if (clusterVersion.CompareTo(new Version(minSupportedVersion)) < 0) { throw new NotSupportedException( string.Format(CultureInfo.InvariantCulture, "Cannot get application history for cluster with version '{0}'. This functionality is supported for clusters with versions {1} and above", cluster.Version, minSupportedVersion)); } }
private static AzureHDInsightClusterConfiguration GetAzureHDInsightClusterConfiguration( ClusterDetails cluster, ClusterContainer clusterCreateDetails) { var azureClusterConfig = new AzureHDInsightClusterConfiguration(); if (clusterCreateDetails.Settings.Core != null && clusterCreateDetails.Settings.Core.Configuration != null) { AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Core.Configuration, "Core"); azureClusterConfig.Core.AddRange( clusterCreateDetails.Settings.Core.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (clusterCreateDetails.Settings.Hive != null && clusterCreateDetails.Settings.Hive.Configuration != null) { AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Hive.Configuration, "Hive"); azureClusterConfig.Hive.AddRange( clusterCreateDetails.Settings.Hive.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (clusterCreateDetails.Settings.Hdfs != null && clusterCreateDetails.Settings.Hdfs.Configuration != null) { AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Hdfs.Configuration, "Hdfs"); azureClusterConfig.Hdfs.AddRange( clusterCreateDetails.Settings.Hdfs.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (clusterCreateDetails.Settings.MapReduce != null && clusterCreateDetails.Settings.MapReduce.Configuration != null) { AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.MapReduce.Configuration, "MapReduce"); azureClusterConfig.MapReduce.AddRange( clusterCreateDetails.Settings.MapReduce.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } if (clusterCreateDetails.Settings.Oozie != null && clusterCreateDetails.Settings.Oozie.Configuration != null) { AssertConfigOptionsAllowed(cluster, clusterCreateDetails.Settings.Oozie.Configuration, "Oozie"); azureClusterConfig.Oozie.AddRange( clusterCreateDetails.Settings.Oozie.Configuration.Select(prop => new KeyValuePair<string, string>(prop.Name, prop.Value))); } return azureClusterConfig; }
/// <summary> /// Initializes a new instance of the HDInsightApplicationHistoryClient class. /// </summary> /// <param name="cluster"> /// Details about the cluster against which the client should be created. /// </param> internal HDInsightApplicationHistoryClient(ClusterDetails cluster) : this(cluster, TimeSpan.FromMinutes(5)) { }
internal static AzureHDInsightJob GetJobWithID(IRunspace runspace, string jobId, ClusterDetails cluster) { IPipelineResult getJobDetailResults = runspace.NewPipeline() .AddCommand(CmdletConstants.GetAzureHDInsightJob) .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl) .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword)) .WithParameter(CmdletConstants.Id, jobId) .Invoke(); return getJobDetailResults.Results.ToEnumerable<AzureHDInsightJob>().FirstOrDefault(); }
/// <summary> /// Initializes a new instance of the ClusterProvisioningStatusEventArgs class. /// </summary> /// <param name="clusterDetails">Details of the cluster being provisioned.</param> /// <param name="clusterState">Current state of the cluster.</param> public ClusterProvisioningStatusEventArgs(ClusterDetails clusterDetails, ClusterState clusterState) { this.Cluster = clusterDetails; this.State = clusterState; }