public static void ListClusters() { var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>() .First(item => item.Thumbprint == Constants.thumbprint); var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert); var client = HDInsightClient.Connect(creds); var clusters = client.ListClusters(); if (clusters.Count > 0) { foreach (var item in clusters) { Console.WriteLine("Cluster: {0}, Nodes: {1}", item.Name, item.ClusterSizeInNodes); } } else { Console.WriteLine("Cluster not found!"); } }
//Create a new HDI cluster public static void CreateCluster() { var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert); var client = HDInsightClient.Connect(creds); //Cluster information var clusterInfo = new ClusterCreateParameters() { Name = "AutomatedHDICluster", Location = "West Europe", DefaultStorageAccountName = Constants.storageAccount, DefaultStorageAccountKey = Constants.storageAccountKey, DefaultStorageContainer = Constants.container, UserName = Constants.clusterUser, Password = Constants.clusterPassword, ClusterSizeInNodes = 2, Version = "2.1" }; Console.Write("Creating cluster..."); var clusterDetails = client.CreateCluster(clusterInfo); Console.Write("Done\n"); ListClusters(); }
public List <ClusterList> GetAsync() { Trace.WriteLine("Entering GetAsync method"); Trace.TraceInformation("Executing GetAsync method at " + DateTime.Now.ToLongTimeString()); Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id string certName = "Azdem187U23713U-1-8-2015-credentials"; //your-subscription-management-cert-name // Create an HDInsight Client X509Store store = new X509Store(StoreName.My); store.Open(OpenFlags.ReadOnly); X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().Single(item => item.FriendlyName == certName); //your friendly name HDInsightCertificateCredential creds = new HDInsightCertificateCredential(subscriptionId, cert); IHDInsightClient client = HDInsightClient.Connect(creds); //var c =Task.Run(() => client.ListClusters()).Result; var cluster = client.ListClusters(); var c1 = new List <ClusterList>(); foreach (var item in cluster) { c1.Add(new ClusterList() { Name = item.Name, Node = item.ClusterSizeInNodes }); } Trace.WriteLine("Leaving GetAsync method"); return(c1); }
public virtual void TestInitialize() { Capabilities.Add("CAPABILITY_FEATURE_CLUSTERS_CONTRACT_1_SDK"); Capabilities.Add("CAPABILITY_FEATURE_CLUSTERS_CONTRACT_2_SDK"); this.DefaultHandler = this.GetDefaultHandler(); this.HdInsightCertCred = new HDInsightCertificateCredential(Guid.Parse(TestSubscription), Certificate); this.Context = new HDInsightSubscriptionAbstractionContext(this.HdInsightCertCred, new CancellationTokenSource().Token); }
public async Task CreateAJobUsingRestClient() { dynamic builder = DynaXmlBuilder.Create(); builder.xmlns("http://schemas.datacontract.org/2004/07/Microsoft.ClusterServices.RDFEProvider.ResourceExtensions.JobSubmission.Models") .xmlns.i("http://www.w3.org/2001/XMLSchema-instance") .xmlns.a("http://schemas.microsoft.com/2003/10/Serialization/Arrays") .ClientJobRequest .b .ApplicationName("pi") .Arguments .b .xmlns.a.@string(16) .xmlns.a.@string(10000) .d .JarFile(Constants.WabsProtocolSchemeName + "[email protected]/hadoop-examples.jar") .JobName("TestJob") .JobType("MapReduce") .OutputStorageLocation("wfoleyeastus") .Parameters .Query .b .at.xmlns.i.nil("true") .d .Resources .d .End(); string payLoad; using (var stream = new MemoryStream()) using (var reader = new StreamReader(stream)) { builder.Save(stream); stream.Position = 0; payLoad = reader.ReadToEnd(); } Assert.IsNotNull(payLoad); string endPoint = @"https://managementnext.rdfetest.dnsdemo4.com"; string dnsName = "wfoley-tortuga-07"; string cloudNamespace = @"hdinsight-wfoley"; var creds = GetCredentials("hadoop"); var subId = creds.SubscriptionId; var x509 = new X509Certificate2(creds.Certificate); //dnsName = "Test-TestJobSubmit-20130624171952-f7e88"; //subId = new Guid("0fec600d-7e0c-4282-ad96-9b515db0471b"); //cloudNamespace = "hdinsight-current"; //endPoint = @"https://umapi.rdfetest.dnsdemo4.com/"; var conCreds = new HDInsightCertificateCredential(subId, x509, new Uri(endPoint), cloudNamespace); var client = ServiceLocator.Instance.Locate <IHDInsightJobSubmissionRestClientFactory>().Create(conCreds, GetAbstractionContext(), false); var result = await client.CreateJob(dnsName, "East US", payLoad); Assert.IsNotNull(result); }
public async Task ListJobsUsingRestClient() { string endPoint = @"https://managementnext.rdfetest.dnsdemo4.com:443"; string cloudNamespace = @"hdinsight-wfoley"; // string cloudNamespace = @"hdinsight"; var creds = GetCredentials("hadoop"); var x509 = new X509Certificate2(creds.Certificate); var conCreds = new HDInsightCertificateCredential(creds.SubscriptionId, x509, new Uri(endPoint), cloudNamespace); var client = ServiceLocator.Instance.Locate<IHDInsightJobSubmissionRestClientFactory>().Create(conCreds, GetAbstractionContext(), false); var result = await client.ListJobs("wfoley-tortuga-07", "East US"); Assert.IsNotNull(result); }
public async Task ListJobsUsingRestClient() { string endPoint = @"https://managementnext.rdfetest.dnsdemo4.com:443"; string cloudNamespace = @"hdinsight-wfoley"; // string cloudNamespace = @"hdinsight"; var creds = GetCredentials("hadoop"); var x509 = new X509Certificate2(creds.Certificate); var conCreds = new HDInsightCertificateCredential(creds.SubscriptionId, x509, new Uri(endPoint), cloudNamespace); var client = ServiceLocator.Instance.Locate <IHDInsightJobSubmissionRestClientFactory>().Create(conCreds, GetAbstractionContext(), false); var result = await client.ListJobs("wfoley-tortuga-07", "East US"); Assert.IsNotNull(result); }
//Delete an existing HDI cluster public static void DeleteCluster() { var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert); var client = HDInsightClient.Connect(creds); Console.Write("Deleting cluster..."); client.DeleteCluster("AutomatedHDICluster"); Console.Write("Done\n"); ListClusters(); }
//List existing HDI clusters public static void ListClusters() { var store = new X509Store(); store.Open(OpenFlags.ReadOnly); var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint); var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert); var client = HDInsightClient.Connect(creds); var clusters = client.ListClusters(); Console.WriteLine("The list of clusters and their details are"); foreach (var item in clusters) { Console.WriteLine("Cluster: {0}, Nodes: {1}, State: {2}, Version: {3}", item.Name, item.ClusterSizeInNodes, item.State, item.Version); } }
private async Task <Dictionary <string, AzureCluster> > ListClusters() { Dictionary <string, AzureCluster> clusterList = new Dictionary <string, AzureCluster>(); HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(_guid, _certificate); IHDInsightClient sClient = HDInsightClient.Connect(sCred); ICollection <ClusterDetails> clusters = await sClient.ListClustersAsync(); foreach (ClusterDetails cluster in clusters) { clusterList.Add(cluster.Name, new AzureCluster(cluster, this)); } return(clusterList); }
public static void TestRunSetup() { AzureSessionInitializer.InitializeAzureSession(); // This is to ensure that all key assemblies are loaded before IOC registration is required. // This is only necessary for the test system as load order is correct for a production run. // types.Add(typeof(GetAzureHDInsightClusterCmdlet)); var cmdletRunManager = ServiceLocator.Instance.Locate <IServiceLocationSimulationManager>(); cmdletRunManager.RegisterType <IAzureHDInsightConnectionSessionManagerFactory, AzureHDInsightConnectionSessionManagerSimulatorFactory>(); cmdletRunManager.RegisterType <IBufferingLogWriterFactory, BufferingLogWriterFactory>(); cmdletRunManager.RegisterType <IAzureHDInsightStorageHandlerFactory, AzureHDInsightStorageHandlerSimulatorFactory>(); cmdletRunManager.RegisterType <IAzureHDInsightSubscriptionResolverFactory, AzureHDInsightSubscriptionResolverSimulatorFactory>(); cmdletRunManager.RegisterType <IAzureHDInsightClusterManagementClientFactory, AzureHDInsightClusterManagementClientSimulatorFactory>(); cmdletRunManager.RegisterType <IAzureHDInsightJobSubmissionClientFactory, AzureHDInsightJobSubmissionClientSimulatorFactory>(); var testManager = new IntegrationTestManager(); AzureSession.Instance.DataStore = new MemoryDataStore(); var profile = new AzureSMProfile(Path.Combine(AzureSession.Instance.ProfileDirectory, AzureSession.Instance.ProfileFile)); AzureSMCmdlet.CurrentProfile = profile; TestCredentials = testManager.GetCredentials("default"); if (TestCredentials == null) { Assert.Inconclusive("No entry was found in the credential config file for the specified test configuration."); } // Sets the certificate var defaultCertificate = new X509Certificate2( Convert.FromBase64String(TestCredentials.Certificate), string.Empty); // Sets the test static properties ClusterPrefix = string.Format("CLITest-{0}", Environment.GetEnvironmentVariable("computername") ?? "unknown"); // Sets the credential objects var tempCredentials = new HDInsightCertificateCredential { SubscriptionId = TestCredentials.SubscriptionId, Certificate = defaultCertificate }; validCredentials = tempCredentials; tempCredentials = new HDInsightCertificateCredential { SubscriptionId = Guid.NewGuid(), Certificate = defaultCertificate }; invalidSubscriptionId = tempCredentials; invalidCertificate = tempCredentials; }
public AzureYarnClient(AzureSubscriptions subscriptions, AzureDfsClient dfsClient, Uri baseUri, string ppmHome, string clusterName = null) { this.dfsClient = dfsClient; this.baseUri = baseUri; this.peloponneseHome = ppmHome; IEnumerable <AzureCluster> clusters = subscriptions.GetClustersAsync().Result; AzureCluster cluster; if (clusterName == null) { if (clusters.Count() != 1) { throw new ArgumentException("A cluster name must be provided if there is not exactly one configured HDInsight cluster.", "clusterName"); } cluster = clusters.Single(); } else { IEnumerable <AzureCluster> matching = clusters.Where(c => c.Name == clusterName); if (matching.Count() == 0) { throw new ArgumentException("Cluster " + clusterName + " not attached to a Powershell subscription or specified manuall", "clusterName"); } cluster = matching.First(); } ClusterName = cluster.Name; SubscriptionId = cluster.SubscriptionId; Guid subscriptionGuid = new Guid(SubscriptionId); HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subscriptionGuid, cluster.Certificate); IHDInsightClient sClient = HDInsightClient.Connect(sCred); credentials = new JobSubmissionCertificateCredential(sCred, ClusterName); JobClient = JobSubmissionClientFactory.Connect(credentials); }
public void TestWithCertificateCredentials() { IHDInsightSubscriptionCredentials credentials = new HDInsightCertificateCredential() { Certificate = new X509Certificate2(IntegrationTestBase.TestCredentials.Certificate), DeploymentNamespace = "hdinsight", Endpoint = new Uri("http://notrdfe.com/"), SubscriptionId = Guid.NewGuid() }; Exception error = null; try { //Should not have token header set when using a certificate IHttpClientAbstraction validAbstraction = ServiceLocator.Instance.Locate <IHDInsightHttpClientAbstractionFactory>().Create(credentials, false); Assert.IsFalse(validAbstraction.RequestHeaders.ContainsKey("Authorization")); } catch (NotSupportedException e) { error = e; } Assert.IsNull(error); }
[Timeout(5 * 60 * 1000)] // ms public async Task ICanPerformA_CreateDeleteContainers_Using_RestClient_ManualEnvironment() { var creds = IntegrationTestBase.GetCredentialsForEnvironmentType(EnvironmentType.Current); if (creds == null) { Assert.Inconclusive("Alternative Azure Endpoint wasn't set up"); } IHDInsightCertificateCredential certCreds = IntegrationTestBase.GetValidCredentials() as IHDInsightCertificateCredential; IHDInsightAccessTokenCredential tokenCreds = IntegrationTestBase.GetValidCredentials() as IHDInsightAccessTokenCredential; IHDInsightSubscriptionCredentials tempCredentials = null; if (certCreds != null) { tempCredentials = new HDInsightCertificateCredential() { SubscriptionId = creds.SubscriptionId, Certificate = certCreds.Certificate }; } else if (tokenCreds != null) { tempCredentials = new HDInsightAccessTokenCredential() { SubscriptionId = creds.SubscriptionId, AccessToken = tokenCreds.AccessToken }; } IHDInsightSubscriptionCredentials credentials = new AlternativeEnvironmentIHDInsightSubscriptionCertificateCredentialsFactory().Create(tempCredentials); var client = new HDInsightManagementRestClient(credentials, GetAbstractionContext(), false); var dnsName = GetRandomClusterName(); var location = "East US"; var subscriptionId = credentials.SubscriptionId; var createPayload = String.Format(CreateContainerGenericRequest, dnsName, location, subscriptionId, Guid.NewGuid()); var xmlReader = new XmlTextReader(new StringReader(createPayload)); var resource = new Resource() { IntrinsicSettings = new[] { new XmlDocument().ReadNode(xmlReader) } }; var result = await client.ListCloudServices(); Assert.IsTrue(!this.ContainsContainer(dnsName, result.Content)); await client.CreateContainer(dnsName, location, resource.SerializeToXml()); result = await client.ListCloudServices(); bool containsContiner = false; while (!containsContiner) { result = await client.ListCloudServices(); containsContiner = this.ContainsContainer(dnsName, result.Content); await Task.Delay(100); } await client.DeleteContainer(dnsName, location); containsContiner = true; while (containsContiner) { result = await client.ListCloudServices(); containsContiner = this.ContainsContainer(dnsName, result.Content); await Task.Delay(100); } }
[Timeout(5 * 60 * 1000)] // ms public async Task ICanPerformA_CreateDeleteContainers_Using_RestClient_ManualEnvironment() { var creds = IntegrationTestBase.GetCredentialsForEnvironmentType(EnvironmentType.Current); if (creds == null) Assert.Inconclusive("Alternative Azure Endpoint wasn't set up"); IHDInsightCertificateCredential certCreds = IntegrationTestBase.GetValidCredentials() as IHDInsightCertificateCredential; IHDInsightAccessTokenCredential tokenCreds = IntegrationTestBase.GetValidCredentials() as IHDInsightAccessTokenCredential; IHDInsightSubscriptionCredentials tempCredentials = null; if (certCreds != null) { tempCredentials = new HDInsightCertificateCredential() { SubscriptionId = creds.SubscriptionId, Certificate = certCreds.Certificate }; } else if (tokenCreds != null) { tempCredentials = new HDInsightAccessTokenCredential() { SubscriptionId = creds.SubscriptionId, AccessToken = tokenCreds.AccessToken }; } IHDInsightSubscriptionCredentials credentials = new AlternativeEnvironmentIHDInsightSubscriptionCertificateCredentialsFactory().Create(tempCredentials); var client = new HDInsightManagementRestClient(credentials, GetAbstractionContext(), false); var dnsName = GetRandomClusterName(); var location = "East US"; var subscriptionId = credentials.SubscriptionId; var createPayload = String.Format(CreateContainerGenericRequest, dnsName, location, subscriptionId, Guid.NewGuid()); var xmlReader = new XmlTextReader(new StringReader(createPayload)); var resource = new Resource() { IntrinsicSettings = new[] { new XmlDocument().ReadNode(xmlReader) } }; var result = await client.ListCloudServices(); Assert.IsTrue(!this.ContainsContainer(dnsName, result.Content)); await client.CreateContainer(dnsName, location, resource.SerializeToXml()); result = await client.ListCloudServices(); bool containsContiner = false; while (!containsContiner) { result = await client.ListCloudServices(); containsContiner = this.ContainsContainer(dnsName, result.Content); await Task.Delay(100); } await client.DeleteContainer(dnsName, location); containsContiner = true; while (containsContiner) { result = await client.ListCloudServices(); containsContiner = this.ContainsContainer(dnsName, result.Content); await Task.Delay(100); } }
public static void TestRunSetup() { // Sets the simulator var runManager = ServiceLocator.Instance.Locate <IServiceLocationSimulationManager>(); ServiceLocator.Instance.Locate <ILogger>().AddWriter(new ConsoleLogWriter(Severity.None, Verbosity.Diagnostic)); runManager.RegisterType <IAsvValidatorClientFactory, AsvValidatorSimulatorClientFactory>(); runManager.RegisterType <IHDInsightManagementRestClientFactory, HDInsightManagementRestSimulatorClientFactory>(); runManager.RegisterType <IRdfeServiceRestClientFactory, RdfeServiceRestSimulatorClientFactory>(); runManager.RegisterType <IAzureHDInsightClusterConfigurationAccessorFactory, AzureHDInsightClusterConfigurationAccessorSimulatorFactory>(); runManager.RegisterInstance <IWabStorageAbstractionFactory>(StorageAccountSimulatorFactory.Instance); runManager.RegisterType <IRemoteHadoopJobSubmissionPocoClientFactory, HadoopJobSubmissionPocoSimulatorClientFactory>(); runManager.RegisterType <IHDInsightJobSubmissionPocoClientFactory, HadoopJobSubmissionPocoSimulatorClientFactory>(); runManager.RegisterType <ISubscriptionRegistrationClientFactory, SubscriptionRegistrationSimulatorClientFactory>(); var testManager = new IntegrationTestManager(); if (!testManager.RunAzureTests()) { Assert.Inconclusive("Azure tests are not configured on this machine."); } IntegrationTestBase.TestCredentials = testManager.GetCredentials("default"); if (IntegrationTestBase.TestCredentials == null) { Assert.Inconclusive("No entry was found in the credential config file for the specified test configuration."); } // Sets the certificate var defaultCertificate = new X509Certificate2(IntegrationTestBase.TestCredentials.Certificate); // Sets the test static properties IntegrationTestBase.ClusterPrefix = string.Format("CLITest-{0}", Environment.GetEnvironmentVariable("computername") ?? "unknown"); // Sets the credential objects var tempCredentials = new HDInsightCertificateCredential() { SubscriptionId = TestCredentials.SubscriptionId, Certificate = defaultCertificate, Endpoint = new Uri(TestCredentials.Endpoint) }; IntegrationTestBase.validCredentials = ServiceLocator.Instance .Locate <IHDInsightSubscriptionCredentialsFactory>() .Create(tempCredentials); tempCredentials = new HDInsightCertificateCredential() { SubscriptionId = Guid.NewGuid(), Certificate = defaultCertificate }; IntegrationTestBase.invalidSubscriptionId = ServiceLocator.Instance .Locate <IHDInsightSubscriptionCredentialsFactory>() .Create(tempCredentials); tempCredentials = new HDInsightCertificateCredential() { SubscriptionId = TestCredentials.SubscriptionId, Certificate = new X509Certificate2(TestCredentials.InvalidCertificate), Endpoint = new Uri(TestCredentials.Endpoint) }; IntegrationTestBase.invalidCertificate = ServiceLocator.Instance .Locate <IHDInsightSubscriptionCredentialsFactory>() .Create(tempCredentials); // Prepares the environment IntegrationTestBase.CleanUpClusters(); }
public static void TestRunSetup() { // This is to ensure that all key assemblies are loaded before IOC registration is required. // This is only necessary for the test system as load order is correct for a production run. // types.Add(typeof(GetAzureHDInsightClusterCmdlet)); var cmdletRunManager = ServiceLocator.Instance.Locate<IServiceLocationSimulationManager>(); cmdletRunManager.RegisterType<IAzureHDInsightConnectionSessionManagerFactory, AzureHDInsightConnectionSessionManagerSimulatorFactory>(); cmdletRunManager.RegisterType<IBufferingLogWriterFactory, BufferingLogWriterFactory>(); cmdletRunManager.RegisterType<IAzureHDInsightStorageHandlerFactory, AzureHDInsightStorageHandlerSimulatorFactory>(); cmdletRunManager.RegisterType<IAzureHDInsightSubscriptionResolverFactory, AzureHDInsightSubscriptionResolverSimulatorFactory>(); cmdletRunManager.RegisterType<IAzureHDInsightClusterManagementClientFactory, AzureHDInsightClusterManagementClientSimulatorFactory>(); cmdletRunManager.RegisterType<IAzureHDInsightJobSubmissionClientFactory, AzureHDInsightJobSubmissionClientSimulatorFactory>(); var testManager = new IntegrationTestManager(); TestCredentials = testManager.GetCredentials("default"); if (TestCredentials == null) { Assert.Inconclusive("No entry was found in the credential config file for the specified test configuration."); } // Sets the certificate var defaultCertificate = new X509Certificate2( Convert.FromBase64String(TestCredentials.Certificate), string.Empty); // Sets the test static properties ClusterPrefix = string.Format("CLITest-{0}", Environment.GetEnvironmentVariable("computername") ?? "unknown"); // Sets the credential objects var tempCredentials = new HDInsightCertificateCredential { SubscriptionId = TestCredentials.SubscriptionId, Certificate = defaultCertificate }; validCredentials = tempCredentials; tempCredentials = new HDInsightCertificateCredential { SubscriptionId = Guid.NewGuid(), Certificate = defaultCertificate }; invalidSubscriptionId = tempCredentials; invalidCertificate = tempCredentials; }
public async Task CreateAJobUsingRestClient() { dynamic builder = DynaXmlBuilder.Create(); builder.xmlns("http://schemas.datacontract.org/2004/07/Microsoft.ClusterServices.RDFEProvider.ResourceExtensions.JobSubmission.Models") .xmlns.i("http://www.w3.org/2001/XMLSchema-instance") .xmlns.a("http://schemas.microsoft.com/2003/10/Serialization/Arrays") .ClientJobRequest .b .ApplicationName("pi") .Arguments .b .xmlns.a.@string(16) .xmlns.a.@string(10000) .d .JarFile(Constants.WabsProtocolSchemeName + "[email protected]/hadoop-examples.jar") .JobName("TestJob") .JobType("MapReduce") .OutputStorageLocation("wfoleyeastus") .Parameters .Query .b .at.xmlns.i.nil("true") .d .Resources .d .End(); string payLoad; using (var stream = new MemoryStream()) using (var reader = new StreamReader(stream)) { builder.Save(stream); stream.Position = 0; payLoad = reader.ReadToEnd(); } Assert.IsNotNull(payLoad); string endPoint = @"https://managementnext.rdfetest.dnsdemo4.com"; string dnsName = "wfoley-tortuga-07"; string cloudNamespace = @"hdinsight-wfoley"; var creds = GetCredentials("hadoop"); var subId = creds.SubscriptionId; var x509 = new X509Certificate2(creds.Certificate); //dnsName = "Test-TestJobSubmit-20130624171952-f7e88"; //subId = new Guid("0fec600d-7e0c-4282-ad96-9b515db0471b"); //cloudNamespace = "hdinsight-current"; //endPoint = @"https://umapi.rdfetest.dnsdemo4.com/"; var conCreds = new HDInsightCertificateCredential(subId, x509, new Uri(endPoint), cloudNamespace); var client = ServiceLocator.Instance.Locate<IHDInsightJobSubmissionRestClientFactory>().Create(conCreds, GetAbstractionContext(), false); var result = await client.CreateJob(dnsName, "East US", payLoad); Assert.IsNotNull(result); }