protected override void ProcessRecord()
{
if (_parametersValid)
{
// generate temp zip file in users %temp% dir and zip the project dir
Random r = new Random();
string zipPath = Path.GetTempPath() + "job" + r.Next(1, 16777216).ToString() + ".zip";
ZipUp(zipPath, _jobDataDirectoryPath);
// create storage helper
var subscription = new AzureSubscription()
{
SubscriptionId = _azureParameters.parameters.SubscriptionID,
ManagementCertificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(
_azureParameters.parameters.PathToManagementCertificate,
_azureParameters.parameters.CertificateEncryptedPassword),
StorageAccount = _azureParameters.parameters.StorageAccountName,
StorageAccountKey = _azureParameters.parameters.StorageAccountKey
};
AzureStorageHelper storage = new AzureStorageHelper(subscription, _clusterName);
AzureCloudController controller = new AzureCloudController(subscription, _azureParameters.parameters.AffinityGroupName);
AzureDynamicCluster cluster = new AzureDynamicCluster(storage, controller);
// upload zip with exectable and data, and run the job (automaticaly from Azure Worker role)
try
{
string jobID = cluster.SubmitJob(new JobItem()
{
CorePerNode = _coresPerNode, //needs to be adjusted in the future
Executable = _executableFileName,
Parameters = _executableArguments,
NumNodes = _azureParameters.parameters.NumberOfNodes,
InfoTag = "test",
JobType = _jobType
},
zipPath);
// delete temp zip file;
File.Delete(zipPath);
// write true to the pipeline indicating successful upload
WriteObject(jobID);
}
catch (Exception ex)
{
// delete temp zip file;
File.Delete(zipPath);
throw ex;
}
}
else
{
WriteObject(null);
}
}
protected override void ProcessRecord()
{
// create storage helper
var subscription = new AzureSubscription()
{
SubscriptionId = _azureParameters.parameters.SubscriptionID,
ManagementCertificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(
_azureParameters.parameters.PathToManagementCertificate,
_azureParameters.parameters.CertificateEncryptedPassword),
StorageAccount = _azureParameters.parameters.StorageAccountName,
StorageAccountKey = _azureParameters.parameters.StorageAccountKey
};
AzureStorageHelper storage = new AzureStorageHelper(subscription, _clusterName);
AzureCloudController controller = new AzureCloudController(subscription, _azureParameters.parameters.AffinityGroupName);
AzureDynamicCluster cluster = new AzureDynamicCluster(storage, controller);
bool downloaded = cluster.DownloadJobResults(_jobID);
// return true if results were downloaded
WriteObject(downloaded);
}
protected override void ProcessRecord()
{
// create storage helper
var subscription = new AzureSubscription()
{
SubscriptionId = _azureParameters.parameters.SubscriptionID,
ManagementCertificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(
_azureParameters.parameters.PathToManagementCertificate,
_azureParameters.parameters.CertificateEncryptedPassword),
StorageAccount = _azureParameters.parameters.StorageAccountName,
StorageAccountKey = _azureParameters.parameters.StorageAccountKey
};
AzureStorageHelper storage = new AzureStorageHelper(subscription, _clusterName);
AzureCloudController controller = new AzureCloudController(subscription, _azureParameters.parameters.AffinityGroupName);
AzureDynamicCluster cluster = new AzureDynamicCluster(storage, controller);
if (!_stopCheckingJobStatus)
{
// return true throught the pipeline if job is finished
WriteObject(cluster.HasJobFinished(_jobID));
}
}
static void Main(string[] args)
{
////////////////////////////////////////////////// CONFIGURATION /////////////////////////////////////////
//Subscription CONFIGURATION
const string subscriptionId = @"";
const string pathToManagementCertificate = @"";
const string mgmtCertificatePassword = @"";
const string storageAccountName = @"";
const string affinityGroupName = @"";
const string storageAccountKey = @"";
//Cluster Configuration
Console.WriteLine("Please enter a name for the cluster (only alphanumeric characters):");
var clusterName = Console.ReadLine();
Console.WriteLine("Please specify the size of the nodes (e.g. Small):");
var nodeSize = Console.ReadLine();
Console.WriteLine("Please enter the number of nodes to create:");
var nodeNumber = Int32.Parse(Console.ReadLine());
Console.WriteLine("Please enter the path to a deployment template");
var pathToTempl = Console.ReadLine();
////////////////////////////////////////////////// CREATION /////////////////////////////////////////
// prepare objects
var subscription = new AzureSubscription()
{
SubscriptionId = subscriptionId,
ManagementCertificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(pathToManagementCertificate, mgmtCertificatePassword),
StorageAccount = storageAccountName,
StorageAccountKey = storageAccountKey
};
var controller = new AzureCloudController(subscription, affinityGroupName);
var storage = new AzureStorageHelper(subscription, clusterName);
var template = new AzureDeploymentTemplate(pathToTempl);
var cluster = new AzureDynamicCluster(storage, controller);
cluster.Initialize(clusterName, storageAccountName, storageAccountKey, new AzureInstanceType()
{
Name = nodeSize
}, nodeNumber, template);
//create Cluster
Console.WriteLine("Creating cluster..");
cluster.CreateCluster();
Console.WriteLine("Creating cluster finished.. Please wait until it is ready (may take about 10 min)");
bool clusterReady = false;
while (!clusterReady)
{
Thread.Sleep(5000);
var state = cluster.GetState();
if (!state.ClusterReady)
{
Console.WriteLine("Cluster not ready:");
foreach (var instanceState in state.InstanceStates)
{
Console.WriteLine("Instance State = " + instanceState.AzureStateName);
Console.WriteLine("Instance State Details = " + instanceState.AzureStateDetails ?? "N/A");
}
Console.WriteLine();
}
else
{
clusterReady = true;
Console.WriteLine("Cluster ready!!");
}
}
////////////////////////////////////////////////// Run Jobs /////////////////////////////////////////
bool exit = false;
while (!exit)
{
Console.WriteLine("If you want to set up a new job, enter \"job\", to delete the cluster enter \"delete\", to see all job states enter \"print\"");
var answer = Console.ReadLine();
if (answer == "delete")
{
exit = true;
continue;
}
if (answer == "print")
{
foreach (JobItem job in cluster.GetJobs())
{
Console.WriteLine("job " + job.RowKey + " " + (job.Finished ? "finished" : "not finished") + " result path = " + (job.Finished ? job.ResultFileName : "N/A"));
}
continue;
}
if (answer == "job")
{
Console.WriteLine("Please enter the path to the ZIP File containing all job data");
string pathToZip = Console.ReadLine();
Console.WriteLine("Please enter the name of the Executable (eg. petsc.exe)");
string exec = Console.ReadLine();
Console.WriteLine("Please enter all command line arguments");
string arguments = Console.ReadLine();
var jobId = cluster.SubmitJob(new JobItem()
{
CorePerNode = 1, //needs to be adjusted in the future
Executable = exec,
Parameters = arguments,
NumNodes = nodeNumber,
InfoTag = "test",
JobType = (int)JobItem.Type.MPI
},
pathToZip);
//get job results
Console.WriteLine(@"Assigned job " + jobId);
while (!cluster.HasJobFinished(jobId))
{
Thread.Sleep(5000);
Console.WriteLine("Job " + jobId + " running/waiting to run...");
}
Console.WriteLine("Downloading job results");
bool downloaded = cluster.DownloadJobResults(jobId);
if (downloaded)
{
Console.WriteLine("Job results stored in file " + cluster.GetJobs().Where(x => x.RowKey == jobId).First().ResultFileName);
}
else
{
Console.Error.WriteLine("Error! Job result was not created");
}
}
}
////////////////////////////////////////////////// CLEAN UP /////////////////////////////////////////
Console.WriteLine("Finished, press enter to delete all deployments");
Console.ReadLine();
cluster.DeleteCluster();
}
protected override void ProcessRecord()
{
// Create Azure Service
if (_parametersValid)
{
// prepare objects
var subscription = new AzureSubscription()
{
SubscriptionId = _subscriptionId,
ManagementCertificate = new System.Security.Cryptography.X509Certificates.X509Certificate2(_pathToManagementCertificate, _mgmtCertificateEncryptedPassword),
StorageAccount = _storageAccountName,
StorageAccountKey = _storageAccountKey
};
AzureCloudController controller = new AzureCloudController(subscription, _affinityGroupName);
// check if azure service name is available
bool available = controller.IsCloudServiceNameAvailable(_azureServiceName);
if (!available)
{
PSArgumentException ex = new PSArgumentException("Azure Service name: " + _azureServiceName + " already exists.");
WriteError(new ErrorRecord(ex, "Argument Exception", ErrorCategory.InvalidData, _azureServiceName));
return;
}
AzureStorageHelper storage = new AzureStorageHelper(subscription, _azureServiceName);
AzureDeploymentTemplate template = new AzureDeploymentTemplate(_pathToDeploymentTemplate);
AzureDynamicCluster cluster = new AzureDynamicCluster(storage, controller);
cluster.Initialize(_azureServiceName, _storageAccountName, _storageAccountKey, new AzureInstanceType()
{
Name = _sizeOfNodes
}, _numberOfNodes, template);
//create Cluster
Console.WriteLine("Creating cluster. Please wait...");
cluster.CreateCluster();
Console.WriteLine("Creating cluster finished.");
Console.WriteLine("Please wait until it is ready (may take up to 10 min.)");
// remember Cursor possition
int left = Console.CursorLeft;
int top = Console.CursorTop;
// initialize stopwatch
Stopwatch sw = new Stopwatch();
sw.Start();
while (!_serviceCreatedAndReady)
{
Thread.Sleep(5000);
var state = cluster.GetState();
if (!state.ClusterReady)
{
Console.CursorLeft = left;
Console.CursorTop = top;
Console.WriteLine("Cluster not ready:");
foreach (var instanceState in state.InstanceStates)
{
Console.WriteLine("Instance State = " + instanceState.AzureStateName + " ");
Console.WriteLine("Instance State Details = " + (instanceState.AzureStateDetails ?? "N/A"));
}
Console.WriteLine(string.Format("Elapsed time: {0}:{1}", Math.Floor(sw.Elapsed.TotalMinutes), sw.Elapsed.ToString("ss")));
}
else
{
_serviceCreatedAndReady = true;
Console.WriteLine();
Console.WriteLine("Cluster ready!");
}
}
sw.Stop();
}
else
{
WriteObject(null);
return;
}
// initialize return object
AzureParameters azureparameters = new AzureParameters()
{
parameters = new AzureParameters.Parameters()
{
SubscriptionID = _subscriptionId,
PathToManagementCertificate = _pathToManagementCertificate,
CertificateEncryptedPassword = _mgmtCertificateEncryptedPassword,
StorageAccountName = _storageAccountName,
AffinityGroupName = _affinityGroupName,
StorageAccountKey = _storageAccountKey,
NumberOfNodes = _numberOfNodes
},
clusterready = _serviceCreatedAndReady
};
WriteObject(azureparameters);
}
