public List <ClusterList> GetAsync()
{
Trace.WriteLine("Entering GetAsync method");
Trace.TraceInformation("Executing GetAsync method at " + DateTime.Now.ToLongTimeString());
Guid subscriptionId = new Guid("44fbb137-edbb-4044-9db9-0e1333e137cf"); //your-subscription-id
string certName = "Azdem187U23713U-1-8-2015-credentials"; //your-subscription-management-cert-name
// Create an HDInsight Client
X509Store store = new X509Store(StoreName.My);
store.Open(OpenFlags.ReadOnly);
X509Certificate2 cert = store.Certificates.Cast <X509Certificate2>().Single(item => item.FriendlyName == certName); //your friendly name
HDInsightCertificateCredential creds = new HDInsightCertificateCredential(subscriptionId, cert);
IHDInsightClient client = HDInsightClient.Connect(creds);
//var c =Task.Run(() => client.ListClusters()).Result;
var cluster = client.ListClusters();
var c1 = new List <ClusterList>();
foreach (var item in cluster)
{
c1.Add(new ClusterList()
{
Name = item.Name, Node = item.ClusterSizeInNodes
});
}
Trace.WriteLine("Leaving GetAsync method");
return(c1);
}
public async Task ICanAddAndDeleteRDPUser()
{
IHDInsightSubscriptionCredentials credentials = IntegrationTestBase.GetValidCredentials();
IHDInsightClient client = HDInsightClient.Connect(credentials);
ClusterCreateParametersV2 clusterCreationDetails = GetRandomCluster();
clusterCreationDetails.Version = "1.6";
ClusterDetails clusterDetails = client.CreateCluster(clusterCreationDetails);
// add a rdp user - rdp is OFF by default, hence we enable it first
string userName = "******";
string password = GetRandomValidPassword();
client.EnableRdp(clusterDetails.Name, clusterDetails.Location, userName, password, DateTime.UtcNow.AddHours(1));
ClusterDetails cluster = await client.GetClusterAsync(clusterDetails.Name);
Assert.AreEqual(userName, cluster.RdpUserName, "Rdp user name has not been updated");
// now disable the rdp user
await client.DisableRdpAsync(clusterDetails.Name, clusterDetails.Location);
cluster = await client.GetClusterAsync(clusterDetails.Name);
Assert.IsFalse(String.IsNullOrEmpty(cluster.Name), "Cluster user name is empty, maybe cluster was not created.");
Assert.IsTrue(String.IsNullOrEmpty(cluster.RdpUserName), "Rdp user name has not been cleared");
// delete the cluster
if (!string.Equals(clusterDetails.Name, IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName))
{
client.DeleteCluster(clusterDetails.Name);
}
}
//Create a new HDI cluster
public static void CreateCluster()
{
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert);
var client = HDInsightClient.Connect(creds);
//Cluster information
var clusterInfo = new ClusterCreateParameters()
{
Name = "AutomatedHDICluster",
Location = "West Europe",
DefaultStorageAccountName = Constants.storageAccount,
DefaultStorageAccountKey = Constants.storageAccountKey,
DefaultStorageContainer = Constants.container,
UserName = Constants.clusterUser,
Password = Constants.clusterPassword,
ClusterSizeInNodes = 2,
Version = "2.1"
};
Console.Write("Creating cluster...");
var clusterDetails = client.CreateCluster(clusterInfo);
Console.Write("Done\n");
ListClusters();
}
public static void ListClusters()
{
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>()
.First(item => item.Thumbprint == Constants.thumbprint);
var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert);
var client = HDInsightClient.Connect(creds);
var clusters = client.ListClusters();
if (clusters.Count > 0)
{
foreach (var item in clusters)
{
Console.WriteLine("Cluster: {0}, Nodes: {1}", item.Name, item.ClusterSizeInNodes);
}
}
else
{
Console.WriteLine("Cluster not found!");
}
}
public IHDInsightClient Create(IHDInsightSubscriptionCredentials credentials, bool ignoreSslErrors)
{
var client = HDInsightClient.Connect(credentials);
client.IgnoreSslErrors = ignoreSslErrors;
return(client);
}
protected static void DeleteClustersWithVersion(IHDInsightCertificateCredential credentials, string version)
{
var client = HDInsightClient.Connect(new HDInsightCertificateCredential(credentials.SubscriptionId, credentials.Certificate));
var clusters = client.ListClusters().Where(cluster => cluster.Version == version).ToList();
Parallel.ForEach(clusters, cluster => client.DeleteCluster(cluster.Name));
}
//Delete an existing HDI cluster
public static void DeleteCluster()
{
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert);
var client = HDInsightClient.Connect(creds);
Console.Write("Deleting cluster...");
client.DeleteCluster("AutomatedHDICluster");
Console.Write("Done\n");
ListClusters();
}
//List existing HDI clusters
public static void ListClusters()
{
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new HDInsightCertificateCredential(Constants.subscriptionId, cert);
var client = HDInsightClient.Connect(creds);
var clusters = client.ListClusters();
Console.WriteLine("The list of clusters and their details are");
foreach (var item in clusters)
{
Console.WriteLine("Cluster: {0}, Nodes: {1}, State: {2}, Version: {3}", item.Name, item.ClusterSizeInNodes, item.State, item.Version);
}
}
private async Task <Dictionary <string, AzureCluster> > ListClusters()
{
Dictionary <string, AzureCluster> clusterList = new Dictionary <string, AzureCluster>();
HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(_guid, _certificate);
IHDInsightClient sClient = HDInsightClient.Connect(sCred);
ICollection <ClusterDetails> clusters = await sClient.ListClustersAsync();
foreach (ClusterDetails cluster in clusters)
{
clusterList.Add(cluster.Name, new AzureCluster(cluster, this));
}
return(clusterList);
}
private void Dispose(bool disposing)
{
if (disposing && !disposed)
{
if (HDInsightClient != null)
{
HDInsightClient.Dispose();
}
if (Context != null)
{
Context.Dispose();
}
}
disposed = true;
}
/// <summary>
/// Connects to HDInsight cluster.
/// </summary>
/// <param name="certificate">The certificate.</param>
/// <param name="subscription">The subscription.</param>
/// <param name="clusterName">Name of the cluster.</param>
/// <param name="storageAccountName">Name of the storage account.</param>
/// <param name="storageAccountKey">The storage account key.</param>
public void Connect(string certificate, string subscription, string clusterName, string storageAccountName, string storageAccountKey)
{
// Obtain the certificate
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().FirstOrDefault(item => string.Compare(item.Thumbprint, certificate, CultureInfo.InvariantCulture, CompareOptions.IgnoreCase) == 0);
if (cert == null)
{
AvroHdiSample.ReportError("Error: Counld not find the certificate on this machine!");
}
// Connect to the cluster using the certificate and the subscription
try
{
this.client = HDInsightClient.Connect(new HDInsightCertificateCredential(new Guid(subscription), cert));
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while connecting to HDInsight service\n" + e);
}
this.cluster = this.client.GetCluster(clusterName);
if (this.cluster == null)
{
AvroHdiSample.ReportError("Error while connecting to cluster: " + clusterName);
}
// Create a job client
this.job = JobSubmissionClientFactory.Connect(
new JobSubmissionCertificateCredential(new Guid(subscription), cert, clusterName));
// Create an Azure storage client
// We will use this client to upload files to Azure storage account
// which is used by HDInsight cluster.
var storageAccount = CloudStorageAccount.Parse(
"DefaultEndpointsProtocol=https;AccountName=" + storageAccountName + ";AccountKey=" + storageAccountKey);
var blobClient = storageAccount.CreateCloudBlobClient();
this.container = blobClient.GetContainerReference(this.cluster.DefaultStorageAccount.Container);
}
public async Task ICanDeleteTheRdpUserTwice()
{
IHDInsightSubscriptionCredentials credentials = IntegrationTestBase.GetValidCredentials();
var client = HDInsightClient.Connect(credentials);
var clusterCreationDetails = GetRandomCluster();
clusterCreationDetails.Version = "1.6";
var clusterDetails = client.CreateCluster(clusterCreationDetails);
await client.DisableRdpAsync(clusterDetails.Name, clusterDetails.Location);
var cluster = await client.GetClusterAsync(clusterDetails.Name);
Assert.IsFalse(String.IsNullOrEmpty(cluster.Name), "Cluster user name is empty, maybe cluster was not created.");
Assert.IsTrue(String.IsNullOrEmpty(cluster.RdpUserName), "Rdp user name has not been cleared");
try
{
await client.DisableRdpAsync(clusterDetails.Name, clusterDetails.Location);
cluster = await client.GetClusterAsync(clusterDetails.Name);
Assert.IsFalse(String.IsNullOrEmpty(cluster.Name), "Cluster user name is empty, maybe cluster was not created.");
Assert.IsTrue(String.IsNullOrEmpty(cluster.RdpUserName), "Rdp user name should still be null");
}
catch (Exception ex)
{
Assert.Fail("Disabling the Rdp user is an idempotent operation but is throwing an error: The following was the error: \r\n{0}", ex);
}
finally
{
// delete the cluster
if (!string.Equals(clusterDetails.Name, IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName))
{
client.DeleteCluster(clusterDetails.Name);
}
}
}
public AzureYarnClient(AzureSubscriptions subscriptions, AzureDfsClient dfsClient, Uri baseUri,
string ppmHome, string clusterName = null)
{
this.dfsClient = dfsClient;
this.baseUri = baseUri;
this.peloponneseHome = ppmHome;
IEnumerable <AzureCluster> clusters = subscriptions.GetClustersAsync().Result;
AzureCluster cluster;
if (clusterName == null)
{
if (clusters.Count() != 1)
{
throw new ArgumentException("A cluster name must be provided if there is not exactly one configured HDInsight cluster.", "clusterName");
}
cluster = clusters.Single();
}
else
{
IEnumerable <AzureCluster> matching = clusters.Where(c => c.Name == clusterName);
if (matching.Count() == 0)
{
throw new ArgumentException("Cluster " + clusterName + " not attached to a Powershell subscription or specified manuall", "clusterName");
}
cluster = matching.First();
}
ClusterName = cluster.Name;
SubscriptionId = cluster.SubscriptionId;
Guid subscriptionGuid = new Guid(SubscriptionId);
HDInsightCertificateCredential sCred = new HDInsightCertificateCredential(subscriptionGuid, cluster.Certificate);
IHDInsightClient sClient = HDInsightClient.Connect(sCred);
credentials = new JobSubmissionCertificateCredential(sCred, ClusterName);
JobClient = JobSubmissionClientFactory.Connect(credentials);
}
public void ListClustersWithAccessToken()
{
string accessToken = "eyiIsIng1dCI6Ik5HVEZ2ZEstZnl0aEV1THdqcHdBSk9NOW4tQSJ9.eyJhdWQiOiJodHRwczovL21hbmFnZW1lbnQuY29yZS53aW5kb3dzLm5ldC8iLCJpc3MiOiJodHRwczovL3N0cy53aW5kb3dzLm5ldC83MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMWRiNDcvIiwiaWF0IjoxMzg0ODAyNzY5LCJuYmYiOjEzODQ4MDI3NjksImV4cCI6MTM4NDgwNjM2OSwidmVyIjoiMS4wIiwidGlkIjoiNzJmOTg4YmYtODZmMS00MWFmLTkxYWItMmQ3Y2QwMTFkYjQ3Iiwib2lkIjoiMzMxMDNiMzEtZTNlNS00NzAxLTk1YmYtOGNjZTA1MTY3MzdjIiwidXBuIjoicGhhbmlyYWpAbWljcm9zb2Z0LmNvbSIsInVuaXF1ZV9uYW1lIjoicGhhbmlyYWpAbWljcm9zb2Z0LmNvbSIsInN1YiI6ImtMYVVCODA2TmxYMjdJSnhka2NtR0FBSUNpSk94T3lVSVZCeVhQMlNIbnciLCJwdWlkIjoiMTAwMzAwMDA4MDFCQ0IxNSIsImZhbWlseV9uYW1lIjoiWWF5YXZhcmFtIE5hcmFzaW1oYSIsImdpdmVuX25hbWUiOiJQaGFuaSBSYWoiLCJhcHBpZCI6IjE5NTBhMjU4LTIyN2ItNGUzMS1hOWNmLTcxNzQ5NTk0NWZjMiIsImFwcGlkYWNyIjoiMCIsInNjcCI6InVzZXJfaW1wZXJzb25hdGlvbiIsImFjciI6IjEifQ.UejKWEUhF35F7NqhfFg73zoXclmrWhCTowpc78TICFOvMzFQYzR8bpBc-NtGOpwcFDN2yIp1eJhPV_QL_gl2Y4HXIl-_ziw_g9KhYeZXTxnhvpS9zquefzTiIoZL4eo-nm6cBkxvHqaeq6P9mkgp6Y3xd9Py1YRkkwKndnoynzWPKHipO0pL_vEpJgMLHzDLkPq2RbFg4ANp35vfxQpPhC0YrbjWYiwsyrsrjYHN9rIkfqyhePcEaLH-jNdZFzL5yJHo3JiDD6CvbeiFpY4_S1y9PEFHSwbXSAmgKBzr9SzBTR0Pm54FMVvsHsYyTdpMS9Pmdzzd9qr3FyXsUrXTaQ";
IHDInsightSubscriptionCredentials credentials = new HDInsightAccessTokenCredential()
{
AccessToken = accessToken,
DeploymentNamespace = "hdinsight",
SubscriptionId = IntegrationTestBase.TestCredentials.SubscriptionId
};
Exception error = null;
try
{
var client = HDInsightClient.Connect(credentials);
var myClusters = client.ListClusters();
}
catch (NotSupportedException e)
{
error = e;
}
Assert.IsNull(error);
}
public async Task ICannotAddAnotherRdpUser()
{
IHDInsightSubscriptionCredentials credentials = IntegrationTestBase.GetValidCredentials();
var client = HDInsightClient.Connect(credentials);
var clusterCreationDetails = GetRandomCluster();
clusterCreationDetails.Version = "1.6";
var clusterDetails = client.CreateCluster(clusterCreationDetails);
await client.DisableRdpAsync(clusterDetails.Name, clusterDetails.Location);
// now add a user
string userName = "******";
string password = GetRandomValidPassword();
await client.EnableRdpAsync(clusterDetails.Name, clusterDetails.Location, userName, password, DateTime.UtcNow.AddHours(1));
try
{
userName = "******";
password = GetRandomValidPassword();
await client.EnableRdpAsync(clusterDetails.Name, clusterDetails.Location, userName, password, DateTime.UtcNow.AddHours(1));
Assert.Fail("This test expected an exception but did not receive one.");
}
catch (HttpLayerException ex)
{
Assert.AreEqual(HttpStatusCode.BadRequest, ex.RequestStatusCode);
}
finally
{
// delete the cluster
if (!string.Equals(clusterDetails.Name, IntegrationTestBase.TestCredentials.WellKnownCluster.DnsName))
{
client.DeleteCluster(clusterDetails.Name);
}
}
}
public IHDInsightClient Create(IHDInsightSubscriptionCredentials credentials)
{
return(HDInsightClient.Connect(credentials));
}