/// <summary>
/// Make a new cluster object representing an Azure HDInsight cluster, reading the details
/// from a subscription stored in the Powershell defaults.
/// </summary>
/// <param name="clusterName">The name of the HDInsight cluster</param>
public DryadLinqAzureCluster(string clusterName)
{
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, "staging"));
}
/// <summary>
/// Make a new cluster object representing an Azure HDInsight cluster, specifying the details
/// manually.
/// </summary>
/// <param name="clusterName">The name of the HDInsight cluster</param>
/// <param name="storageAccount">The storage account to use for staging job resources</param>
/// <param name="storageContainer">The storage account container to use for staging job resources</param>
/// <param name="storageKey">The storage account key, which will be looked up in the subscription if null</param>
public DryadLinqAzureCluster(string clusterName, string storageAccount, string storageContainer, string storageKey = null)
{
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
if (storageKey != null)
{
_azureSubscriptions.AddAccount(storageAccount, storageKey);
}
_cluster = _azureSubscriptions.GetClusterAsync(clusterName)
.ContinueWith(t => { t.Result.SetStorageAccount(storageAccount, storageKey); return(t.Result); });
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
}
/// <summary>
/// Make a new cluster object representing an Azure HDInsight cluster, specifying the details
/// manually
/// </summary>
/// <param name="clusterName">The name of the HDInsight cluster</param>
/// <param name="subscriptionId">The ID of the subscription to fetch cluster details from</param>
/// <param name="certificate">The certificate associated with the subscription</param>
/// <param name="storageAccount">The storage account to use for staging job resources</param>
/// <param name="storageContainer">The storage account container to use for staging job resources</param>
/// <param name="storageKey">The storage account key, which will be looked up in the subscription if null</param>
public DryadLinqAzureCluster(string clusterName, string subscriptionId, X509Certificate2 certificate,
string storageAccount, string storageContainer, string storageKey = null)
{
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
if (storageKey != null)
{
_azureSubscriptions.AddAccount(storageAccount, storageKey);
}
_azureSubscriptions.AddCluster(clusterName, storageAccount, storageKey, subscriptionId, certificate);
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(
c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
}
public Config(string clusterName, string container, string logPath)
{
cluster = clusterName;
AzureSubscriptions azs = new AzureSubscriptions();
Task <AzureCluster> clusterTask = azs.GetClusterAsync(clusterName);
clusterTask.Wait();
accountName = clusterTask.Result.StorageAccount;
storageKey = clusterTask.Result.StorageKey;
containerName = container;
testLogPath = logPath;
}
static int Run(string[] args)
{
Flags.Parse(ConfigurationManager.AppSettings);
args = Flags.Parse(args);
if (ShowHelp.BooleanValue || args.Length == 0)
{
Console.Error.WriteLine(Usage);
return(0);
}
if (!File.Exists(args[0]))
{
Console.Error.WriteLine("Error: Naiad program {0} does not exist.", args[0]);
Console.Error.WriteLine(Usage);
return(-1);
}
AzureSubscriptions subscriptionManagement = new AzureSubscriptions();
if (AzureSubscriptionId.IsSet && AzureCertificateThumbprint.IsSet)
{
subscriptionManagement.AddSubscription(AzureSubscriptionId.StringValue, AzureCertificateThumbprint.StringValue);
}
string clusterName = null;
if (AzureClusterName.IsSet)
{
clusterName = AzureClusterName.StringValue;
if (AzureStorageAccountName.IsSet && AzureStorageAccountKey.IsSet)
{
subscriptionManagement.SetClusterAccountAsync(clusterName, AzureStorageAccountName.StringValue, AzureStorageAccountKey.StringValue).Wait();
}
}
else
{
IEnumerable <AzureCluster> clusters = subscriptionManagement.GetClusters();
if (clusters.Count() == 1)
{
clusterName = clusters.Single().Name;
}
else
{
Console.Error.WriteLine("Error: Cluster name must be specified unless there is a single configured cluster in default and supplied subscriptions");
Console.Error.WriteLine(Usage);
return(-1);
}
}
AzureCluster cluster;
try
{
cluster = subscriptionManagement.GetClusterAsync(clusterName).Result;
}
catch (Exception)
{
Console.Error.WriteLine("Error: Failed to find cluster " + clusterName + " in default or supplied subscriptions");
Console.Error.WriteLine(Usage);
return(-1);
}
if (cluster == null)
{
Console.Error.WriteLine("Error: Failed to find cluster {0} in default or supplied subscriptions", clusterName);
Console.Error.WriteLine(Usage);
return(-1);
}
string containerName = "staging";
if (AzureStorageContainerName.IsSet)
{
containerName = AzureStorageContainerName.StringValue;
}
// The args are augmented with an additional setting containing the Azure connection string.
args = args.Concat(new string[] { "--addsetting", "Microsoft.Research.Naiad.Cluster.Azure.DefaultConnectionString", string.Format("\"DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}\"", cluster.StorageAccount.Split('.').First(), cluster.StorageKey) }).ToArray();
Console.Error.WriteLine("Submitting job with args: {0}", string.Join(" ", args));
AzureDfsClient azureDfs = new AzureDfsClient(cluster.StorageAccount, cluster.StorageKey, containerName);
AzureYarnClient azureYarn = new AzureYarnClient(subscriptionManagement, azureDfs, ConfigHelpers.GetPPMHome(null), clusterName);
AzureYarnSubmission submission = new AzureYarnSubmission(azureDfs, azureYarn, NumHosts, args);
submission.Submit();
return(submission.Join());
}