/// <summary>
/// Initializes a new instance of the DryadLinqContext class for a specified cluster.
/// </summary>
/// <param name="cluster">The cluster to run DryadLINQ jobs</param>
public DryadLinqContext(DryadLinqCluster cluster)
{
// Verify that the head node is set
if (String.IsNullOrEmpty(cluster.HeadNode))
{
throw new DryadLinqException(DryadLinqErrorCode.ClusterNameMustBeSpecified,
SR.ClusterNameMustBeSpecified);
}
this.CommonInit();
this._platformKind = cluster.Kind;
this._headNode = cluster.HeadNode;
this._clusterDetails = cluster;
if (cluster.Kind == DryadLinq.PlatformKind.YARN_NATIVE)
{
this._storageSetScheme = DataPath.HDFS_URI_SCHEME;
// make an Azure subscriptions object just in case we want to access azure streams from the native yarn cluster
this._azureSubscriptions = new AzureSubscriptions();
}
else if (cluster.Kind == DryadLinq.PlatformKind.YARN_AZURE)
{
this._storageSetScheme = DataPath.AZUREBLOB_URI_SCHEME;
DryadLinqAzureCluster azureCluster = cluster as DryadLinqAzureCluster;
this._azureSubscriptions = azureCluster.Subscriptions;
}
}
public async Task LoadSubscriptions()
{
this.SubscriptionsGridView.DataSource = null;
this.comboResourceGroup.DataSource = null;
PaaSController backend = new PaaSController(GetContext());
try
{
string content = await backend.GetSubscriptions();
AzureSubscriptions subscriptions = JsonConvert.DeserializeObject <AzureSubscriptions>(content);
if (subscriptions == null || subscriptions.Value == null || subscriptions.Value.Count() == 0)
{
MessageBox.Show("No subscriptions found.");
return;
}
this.SubscriptionsGridView.DataSource = subscriptions.Value;
this.SubscriptionsGridView.CurrentCell = this.SubscriptionsGridView[1, 0];
}
catch (Exception e)
{
MessageBox.Show($"Failed to get subscriptions: {e.Message}");
return;
}
await LoadResourceGroups();
}
/// <summary>
/// Make a new cluster object representing an Azure HDInsight cluster, reading the details
/// from a subscription stored in the Powershell defaults.
/// </summary>
/// <param name="clusterName">The name of the HDInsight cluster</param>
public DryadLinqAzureCluster(string clusterName)
{
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, "staging"));
}
public ActionResult Index()
{
var subscriptions = AzureSubscriptions.ListForOverview();
var databases = AzureSQLDatabases.ListForOverview();
return(View(new OverviewViewModel {
Services = subscriptions.Concat(databases).OrderBy(_ => _.Name).ToList()
}));
}
/// <summary>
/// Make a new cluster object representing an Azure HDInsight cluster, specifying the details
/// manually.
/// </summary>
/// <param name="clusterName">The name of the HDInsight cluster</param>
/// <param name="storageAccount">The storage account to use for staging job resources</param>
/// <param name="storageContainer">The storage account container to use for staging job resources</param>
/// <param name="storageKey">The storage account key, which will be looked up in the subscription if null</param>
public DryadLinqAzureCluster(string clusterName, string storageAccount, string storageContainer, string storageKey = null)
{
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
if (storageKey != null)
{
_azureSubscriptions.AddAccount(storageAccount, storageKey);
}
_cluster = _azureSubscriptions.GetClusterAsync(clusterName)
.ContinueWith(t => { t.Result.SetStorageAccount(storageAccount, storageKey); return(t.Result); });
_dfsClient = _cluster.ContinueWith(c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
}
/// <summary>
/// Make a new cluster object representing an Azure HDInsight cluster, specifying the details
/// manually
/// </summary>
/// <param name="clusterName">The name of the HDInsight cluster</param>
/// <param name="subscriptionId">The ID of the subscription to fetch cluster details from</param>
/// <param name="certificate">The certificate associated with the subscription</param>
/// <param name="storageAccount">The storage account to use for staging job resources</param>
/// <param name="storageContainer">The storage account container to use for staging job resources</param>
/// <param name="storageKey">The storage account key, which will be looked up in the subscription if null</param>
public DryadLinqAzureCluster(string clusterName, string subscriptionId, X509Certificate2 certificate,
string storageAccount, string storageContainer, string storageKey = null)
{
// start fetching details about the subscriptions, available clusters, etc.
_azureSubscriptions = new AzureSubscriptions();
if (storageKey != null)
{
_azureSubscriptions.AddAccount(storageAccount, storageKey);
}
_azureSubscriptions.AddCluster(clusterName, storageAccount, storageKey, subscriptionId, certificate);
_cluster = _azureSubscriptions.GetClusterAsync(clusterName);
_dfsClient = _cluster.ContinueWith(
c => new AzureDfsClient(c.Result.StorageAccount, c.Result.StorageKey, storageContainer));
}
public Config(string clusterName, string container, string logPath)
{
cluster = clusterName;
AzureSubscriptions azs = new AzureSubscriptions();
Task <AzureCluster> clusterTask = azs.GetClusterAsync(clusterName);
clusterTask.Wait();
accountName = clusterTask.Result.StorageAccount;
storageKey = clusterTask.Result.StorageKey;
containerName = container;
testLogPath = logPath;
}
public ActionResult AddAzureSubscriptionStep1(CreateManagementCertificateCommand cmd, string sessionId)
{
if (!ModelState.IsValid)
{
return(View(cmd));
}
var password = Guid.NewGuid().ToString();
var pfx = WadGraphEs.X509Tools.GenerateCertificate.GeneratePfx(cmd.CertificateName, password);
AzureSubscriptions.StorePfxForSession(sessionId, pfx, password);
return(RedirectToAction("AddAzureSubscriptionStep2", new { sessionId = sessionId }));
}
/// <summary>
/// Initializes a new instance of the DryadLinqContext class for local execution.
/// </summary>
/// <param name="numProcesses">The number of local worker processes that should be started.</param>
/// <param name="storageSetScheme">The default scheme for storage. Defaults to partitioned file</param>
public DryadLinqContext(int numProcesses, string storageSetScheme = null)
{
this.CommonInit();
this._platformKind = PlatformKind.LOCAL;
this._localExecution = true;
this._headNode = "LocalExecution";
this._storageSetScheme = storageSetScheme;
if (String.IsNullOrEmpty(this._storageSetScheme))
{
this._storageSetScheme = DataPath.PARTFILE_URI_SCHEME;
}
this._jobMinNodes = numProcesses;
// make an Azure subscriptions object just in case we want to access azure streams from local execution
this._azureSubscriptions = new AzureSubscriptions();
}
public async Task <ICollection <ChartInfo> > ListAllCharts()
{
var subscriptionCharts = AzureSubscriptions.ListAllCharts();
var sqlCharts = AzureSQLDatabases.ListAllCharts();
return((await subscriptionCharts).Concat(sqlCharts)
.Concat(new [] { new ChartInfo {
Name = "Dummy",
ResourceName = "DummyResourceName",
ResourceType = "DummyResourceType",
ServiceName = "Dummy Service",
ServiceType = "DummyService",
Uri = "wadgraphes://dummy"
} })
.ToList());
}
public async Task <IEnumerable <UsageObject> > Get()
{
_logger.Info("Getting usages");
try {
var tasks = AzureSubscriptions.ListAll().Select(GetUsageForSubscription);
var syncResults = GetSQLDatabaseUsages();
var asyncResults = await Task.WhenAll(tasks);
return(new [] { new UsageObject {
GraphiteCounterName = new GraphiteCounterName("WadGraphEs.Diagnostics.Proxy.TimeOfDay").ToString(),
Timestamp = DateTime.UtcNow.ToString("o"),
Value = DateTime.UtcNow.TimeOfDay.TotalSeconds
} }.Concat(asyncResults.SelectMany(_ => _)).Concat(syncResults).ToList());
}
catch (Exception e) {
_logger.ErrorException("Error getting usages", e);
throw;
}
}
static void Main(string[] args)
{
Uri uri = new Uri(args[0]);
string account, key, container, blob;
Utils.FromAzureUri(uri, out account, out key, out container, out blob);
if (key == null)
{
AzureSubscriptions subs = new AzureSubscriptions();
key = subs.GetAccountKeyAsync(account).Result;
uri = Utils.ToAzureUri(account, container, blob, null, key);
}
Stream log = new AzureLogReaderStream(uri);
if (args.Length > 1 && args[1] == "-tail")
{
Tail(log);
}
else
{
log.CopyTo(Console.OpenStandardOutput());
}
}
public ActionResult AddAzureSubscriptionStep3(EnterSubscriptionIdCommand cmd)
{
if (!ModelState.IsValid)
{
return(View(cmd));
}
var exceptions = AzureSubscriptions.TestConnection(cmd.SessionId, cmd.AzureSubscriptionId);
if (exceptions.Any())
{
foreach (var exc in exceptions)
{
ModelState.AddModelError("test-configuration", exc);
}
return(View(cmd));
}
AzureSubscriptions.AddAzureSubscriptionIdForSession(cmd.SessionId, cmd.AzureSubscriptionId);
return(RedirectToAction("AddAzureSubscriptionStep4", new { sessionId = cmd.SessionId }));
}
static int Run(string[] args)
{
Flags.Parse(ConfigurationManager.AppSettings);
args = Flags.Parse(args);
if (ShowHelp.BooleanValue || args.Length == 0)
{
Console.Error.WriteLine(Usage);
return(0);
}
if (!File.Exists(args[0]))
{
Console.Error.WriteLine("Error: Naiad program {0} does not exist.", args[0]);
Console.Error.WriteLine(Usage);
return(-1);
}
AzureSubscriptions subscriptionManagement = new AzureSubscriptions();
if (AzureSubscriptionId.IsSet && AzureCertificateThumbprint.IsSet)
{
subscriptionManagement.AddSubscription(AzureSubscriptionId.StringValue, AzureCertificateThumbprint.StringValue);
}
string clusterName = null;
if (AzureClusterName.IsSet)
{
clusterName = AzureClusterName.StringValue;
if (AzureStorageAccountName.IsSet && AzureStorageAccountKey.IsSet)
{
subscriptionManagement.SetClusterAccountAsync(clusterName, AzureStorageAccountName.StringValue, AzureStorageAccountKey.StringValue).Wait();
}
}
else
{
IEnumerable <AzureCluster> clusters = subscriptionManagement.GetClusters();
if (clusters.Count() == 1)
{
clusterName = clusters.Single().Name;
}
else
{
Console.Error.WriteLine("Error: Cluster name must be specified unless there is a single configured cluster in default and supplied subscriptions");
Console.Error.WriteLine(Usage);
return(-1);
}
}
AzureCluster cluster;
try
{
cluster = subscriptionManagement.GetClusterAsync(clusterName).Result;
}
catch (Exception)
{
Console.Error.WriteLine("Error: Failed to find cluster " + clusterName + " in default or supplied subscriptions");
Console.Error.WriteLine(Usage);
return(-1);
}
if (cluster == null)
{
Console.Error.WriteLine("Error: Failed to find cluster {0} in default or supplied subscriptions", clusterName);
Console.Error.WriteLine(Usage);
return(-1);
}
string containerName = "staging";
if (AzureStorageContainerName.IsSet)
{
containerName = AzureStorageContainerName.StringValue;
}
// The args are augmented with an additional setting containing the Azure connection string.
args = args.Concat(new string[] { "--addsetting", "Microsoft.Research.Naiad.Cluster.Azure.DefaultConnectionString", string.Format("\"DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}\"", cluster.StorageAccount.Split('.').First(), cluster.StorageKey) }).ToArray();
Console.Error.WriteLine("Submitting job with args: {0}", string.Join(" ", args));
AzureDfsClient azureDfs = new AzureDfsClient(cluster.StorageAccount, cluster.StorageKey, containerName);
AzureYarnClient azureYarn = new AzureYarnClient(subscriptionManagement, azureDfs, ConfigHelpers.GetPPMHome(null), clusterName);
AzureYarnSubmission submission = new AzureYarnSubmission(azureDfs, azureYarn, NumHosts, args);
submission.Submit();
return(submission.Join());
}
public ActionResult AddAzureSubscriptionStep4(FinishAddingAzureSubscription cmd)
{
AzureSubscriptions.Handle(cmd);
return(RedirectToRoute("Home"));
}
public ActionResult AddAzureSubscriptionStep4(string sessionId)
{
var finishCommand = AzureSubscriptions.CreateFinishCommandForSession(sessionId);
return(View(finishCommand));
}
public ActionResult DownloadCertificate(string sessionId)
{
var cert = AzureSubscriptions.GetCertificateForSession(sessionId);
return(File(cert, "application/octet-stream", "management.cer"));
}