/// <summary>
/// Updates the status of the masurca assembler. If it has either errored out (-1) or is completed (last step), it will not change the status.
/// </summary>
/// <param name="client">The current SSH client session.</param>
/// <param name="genomeModel">The model of the particular job.</param>
private static void CheckMasurcaStep(SshClient client, GenomeModel genomeModel)
{
if (string.IsNullOrEmpty(ErrorHandling.error) &&
!genomeModel.MasurcaCurrentStep.Equals(-1) &&
!genomeModel.MasurcaCurrentStep.Equals(StepDescriptions.GetMasurcaStepList().Last().step))
{
if (LinuxCommands.DirectoryHasFiles(client, Accessors.GetMasurcaOutputPath(genomeModel.Seed)))
{
int currentMasurcaStep = LinuxCommands.GetCurrentStep(client, Accessors.GetMasurcaOutputPath(genomeModel.Seed), StepDescriptions.GetMasurcaStepList());
// Provided we didn't encounter an error, set the status of masurca and the job.
if (currentMasurcaStep != -1)
{
genomeModel.MasurcaCurrentStep = currentMasurcaStep;
genomeModel.MasurcaStatus = StepDescriptions.GetCurrentStepDescription(StepDescriptions.GetMasurcaStepList(), currentMasurcaStep);
}
else
{
StepDescriptions.SetMasurcaError(client, genomeModel);
}
}
// Either masurca hasn't started or it has but no files have been created yet.
else
{
genomeModel.MasurcaCurrentStep = 1;
genomeModel.MasurcaStatus = StepDescriptions.GetCurrentStepDescription(StepDescriptions.GetMasurcaStepList(), 1);
}
}
}
/// <summary>
/// We need to verify that a user has sufficient permissions in order to run the assemblers.
/// </summary>
/// <param name="SSHUser">The SSH username of the user.</param>
/// <param name="SSHPass">The SSH password of the user.</param>
/// <returns>Returns a boolean value representing whether or not they have sufficient permissions.</returns>
public static bool VerifyPermissions(string SSHUser, string SSHPass)
{
using (var client = new SshClient(Accessors.BD_IP, SSHUser, SSHPass))
{
try
{
client.Connect();
LinuxCommands.CreateDirectory(client, Accessors.VERIFY_PERMISSIONS_TEST_DIR);
if (!string.IsNullOrEmpty(ErrorHandling.error))
{
ErrorHandling.error = "You do not have sufficient permissions to write to the proper directories. Please contact the BigDog Linux team about addressing this problem.";
return(false);
}
else
{
// We want to remove the directory we just created as a test. We recursively force the deletion.
LinuxCommands.RemoveFile(client, Accessors.VERIFY_PERMISSIONS_TEST_DIR, "-rf");
return(true);
}
}
// SSH Connection couldn't be established.
catch (SocketException e)
{
ErrorHandling.error = "The SSH connection couldn't be established. " + e.Message;
return(false);
}
// Authentication failure.
catch (SshAuthenticationException e)
{
ErrorHandling.error = "The credentials were entered incorrectly. " + e.Message;
return(false);
}
// The SSH connection was dropped.
catch (SshConnectionException e)
{
ErrorHandling.error = "The connection was terminated unexpectedly. " + e.Message;
return(false);
}
catch (Exception e)
{
ErrorHandling.error = "There was an uncaught exception. " + e.Message;
return(false);
}
}
}
/// <summary>
/// Verifies that a particular user has sufficient quota in order to run the assemblers.
/// </summary>
/// <param name="SSHUser">The SSH username of the user.</param>
/// <param name="SSHPass">The SSH password of the user.</param>
/// <param name="quotaAmount">The amount of quota in GB that the user has. This is sent out of the method.</param>
/// <returns>Returns a boolean value representing whether or not they have enough space.</returns>
public static bool VerifyQuota(string SSHUser, string SSHPass)
{
using (var client = new SshClient(Accessors.BD_IP, SSHUser, SSHPass))
{
try
{
client.Connect();
int userQuota = LinuxCommands.GetQuota(client, Accessors.MINIMUM_QUOTA);
// If they have less than 'minQuota' then we return an error telling them the problem and how to rectify it.
if (userQuota < Accessors.MINIMUM_QUOTA)
{
ErrorHandling.error = "You do not have the requisite amount of disk space (" + Accessors.MINIMUM_QUOTA + "Gb) for us to safely run a general assembly "
+ "job. Please contact the BigDog admin team to increase your quota. You currently have " + userQuota + "Gb space to use.";
return(false);
}
// They have at least the minimum quota.
else
{
return(true);
}
}
// SSH Connection couldn't be established.
catch (SocketException e)
{
ErrorHandling.error = "The SSH connection couldn't be established. " + e.Message;
return(false);
}
// Authentication failure.
catch (SshAuthenticationException e)
{
ErrorHandling.error = "The credentials were entered incorrectly. " + e.Message;
return(false);
}
// The SSH connection was dropped.
catch (SshConnectionException e)
{
ErrorHandling.error = "The connection was terminated unexpectedly. " + e.Message;
return(false);
}
catch (Exception e)
{
ErrorHandling.error = "There was an uncaught exception. " + e.Message;
return(false);
}
}
}
/// <summary>
/// Tests whether the URLs entered by the user in the wizard are connectable.
/// </summary>
/// <param name="sshUser">The user's SSH username.</param>
/// <param name="sshPass">The user's SSH paswword.</param>
/// <param name="dataSource">The list of URLs separated by commas.</param>
/// <returns>Returns a null string if the URL is connectable. Otherwise it will return the URL that is malfunctioning.</returns>
public static string TestJobUrls(string sshUser, string sshPass, string dataSource)
{
using (var client = new SshClient(Accessors.BD_IP, sshUser, sshPass))
{
client.Connect();
List <string> urlList = ParseUrlString(dataSource);
foreach (var url in urlList)
{
if (!LinuxCommands.CheckDataAvailability(client, url))
{
return(url);
}
}
return("");
}
}
/// <summary>
/// Creates all the necessary folders, downloads the config scripts, and adds the job to the scheduler on BigDog.
/// </summary>
/// <returns>Returns true only if a job gets successfully added to SGE.</returns>
public bool CreateJob()
{
// The init.sh script will contain all the basic logic to download the data and initiate the job on the assembler(s).
using (var client = new SshClient(Accessors.BD_IP, genomeModel.SSHUser, genomeModel.SSHPass))
{
// Set defaults
//Accessors.masterPath = "/share/scratch/bioinfo/" + HelperMethods.GetUsername();
string node = Accessors.BD_COMPUTE_NODE1; // default
string wgetLogParameter = "--output-file=" + Accessors.GetJobLogPath(seed) + "wget.error";
string initPath = Accessors.GetJobConfigPath(seed) + "init.sh";
string masurcaPath = Accessors.GetJobConfigPath(seed) + "masurca_config.txt";
string jobName = HelperMethods.GetUsername() + "-" + seed;
try
{
client.Connect();
if (NoError())
{
LinuxCommands.CreateDirectory(client, Accessors.USER_ROOT_JOB_DIRECTORY, "-p");
}
if (NoError())
{
LinuxCommands.CreateDirectory(client, Accessors.GetJobPath(seed), "-p");
}
if (NoError())
{
LinuxCommands.CreateDirectory(client, Accessors.GetJobDataPath(seed), "-p");
}
if (NoError())
{
LinuxCommands.CreateDirectory(client, Accessors.GetJobConfigPath(seed), "-p");
}
if (NoError())
{
LinuxCommands.CreateDirectory(client, Accessors.GetJobOutputPath(seed), "-p");
}
if (NoError())
{
// Create subdirectories in the output directory to separate the outputs.
if (genomeModel.UseMasurca)
{
LinuxCommands.CreateDirectory(client, Accessors.GetMasurcaOutputPath(seed), "-p");
}
if (genomeModel.UseSGA)
{
LinuxCommands.CreateDirectory(client, Accessors.GetSgaOutputPath(seed), "-p");
}
if (genomeModel.UseWGS)
{
LinuxCommands.CreateDirectory(client, Accessors.GetWgsOutputPath(seed), "-p");
}
}
if (NoError())
{
LinuxCommands.CreateDirectory(client, Accessors.GetJobLogPath(seed), "-p");
}
if (NoError())
{
LinuxCommands.DownloadFile(client, initPath, InitConfigURL, wgetLogParameter);
}
if (NoError())
{
LinuxCommands.RunDos2Unix(client, initPath);
}
if (NoError())
{
LinuxCommands.DownloadFile(client, masurcaPath, MasurcaConfigURL, wgetLogParameter);
}
if (NoError())
{
LinuxCommands.RunDos2Unix(client, masurcaPath);
}
if (NoError())
{
LinuxCommands.ChangePermissions(client, Accessors.GetJobPath(seed), "777", "-R");
}
if (NoError())
{
// So COMPUTENODE2 has a smaller load, we want to use that instead.
if (LinuxCommands.GetNodeLoad(client, Accessors.BD_COMPUTE_NODE1) > LinuxCommands.GetNodeLoad(client, Accessors.BD_COMPUTE_NODE2))
{
node = Accessors.BD_COMPUTE_NODE2;
}
else
{
node = Accessors.BD_COMPUTE_NODE1;
}
}
if (NoError())
{
LinuxCommands.AddJobToScheduler(client, Accessors.GetJobDataPath(seed), Accessors.GetJobLogPath(seed), node, jobName, initPath);
}
if (NoError())
{
genomeModel.SGEJobId = LinuxCommands.SetJobNumber(client, genomeModel.SSHUser, jobName);
}
// There were no errors.
if (NoError())
{
return(true);
}
else
{
return(false);
}
}
// SSH Connection couldn't be established.
catch (SocketException e)
{
ErrorHandling.error = "The SSH connection couldn't be established. " + e.Message;
return(false);
}
// Authentication failure.
catch (SshAuthenticationException e)
{
ErrorHandling.error = "The credentials were entered incorrectly. " + e.Message;
return(false);
}
// The SSH connection was dropped.
catch (SshConnectionException e)
{
ErrorHandling.error = "The connection was terminated unexpectedly. " + e.Message;
return(false);
}
catch (Exception e)
{
ErrorHandling.error = "There was an uncaught exception. " + e.Message;
return(false);
}
}
}
/// <summary>
/// Updates the status of a single job. But it does not perform the upload if that needs to happen.
/// </summary>
/// <param name="genomeModel">The model of the particular job.</param>
protected internal static void UpdateStatus(GenomeModel genomeModel)
{
using (var client = new SshClient(CreatePrivateKeyConnectionInfo()))
{
try
{
client.Connect();
/// TODO: Modify the code to skip entire sections if they have already been completed. This will be based off the CURRENT STEP stored in the model data.
using (GenomeAssemblyDbContext db = new GenomeAssemblyDbContext())
{
bool continueUpdate = true; // Determines whether we will continue checking the job status.
bool DEBUG_MODE = false; // Debug mode to skip some assembler steps.
bool outOfRange = false; // If the overall step is out of bounds, then we set this to true to attempt a correction.
ErrorHandling.error = ""; // Reset the errror flag.
while (continueUpdate && ErrorHandling.NoError())
{
// Depending on the current step, this switch will determine if the state of the job needs to change.
switch (genomeModel.OverallCurrentStep)
{
// Queued step
case 1:
{
if (DEBUG_MODE)
{
genomeModel.NextStep();
break;
}
if (LinuxCommands.IsProcessRunning(client, "conversionScript.sh"))
{
genomeModel.NextStep();
}
// If a conversion was never run or if we missed it, then check if the job has already started.
else if (LinuxCommands.JobRunningAlt(client, genomeModel.SSHUser))
{
genomeModel.NextStep();
}
else
{
continueUpdate = false;
}
break;
}
// Data conversion step
case 2:
{
if (DEBUG_MODE)
{
genomeModel.NextStep();
break;
}
if (LinuxCommands.DirectoryHasFiles(client, Accessors.GetMasurcaOutputPath(genomeModel.Seed)))
{
genomeModel.NextStep();
}
else
{
continueUpdate = false;
}
break;
}
// Running assemblers step
case 3:
{
if (DEBUG_MODE)
{
genomeModel.NextStep();
break;
}
if (genomeModel.UseMasurca)
{
CheckMasurcaStep(client, genomeModel);
}
if (genomeModel.UseSGA)
{
}
if (genomeModel.UseWGS)
{
}
if (genomeModel.IsAssemblyFinished())
{
genomeModel.NextStep();
}
else
{
continueUpdate = false;
}
break;
}
// Data analysis step
case 4:
{
// Until data analysis is implemented, we skip the step.
genomeModel.NextStep();
break;
//if (LinuxCommands.IsProcessRunning(client, "dataAnalysis.sh"))
// continueUpdate = false;
//else
//{
// // Has it finished?
// if (LinuxCommands.FileExists(client, Accessors.GetJobOutputPath(genomeModel.Seed) + "dataAnalysisResult"))
// genomeModel.NextStep();
// else
// LinuxCommands.RunDataAnalysis(client);
//}
//break;
}
// TODO: Create a more robust method in checking for a completed upload. Maybe connect to the FTP and compare file sizes and see if they are close.
// Uploading Data step
case 5:
{
//if (LinuxCommands.IsJobUploading(client, Accessors.USER_ROOT_JOB_DIRECTORY, Accessors.GetCompressedDataPath(genomeModel.Seed)))
// continueUpdate = false;
//else if (LinuxCommands.FileExists(client, Accessors.GetCompressedDataPath(genomeModel.Seed)))
// genomeModel.NextStep();
//else
//{
LinuxCommands.UploadJobData(client, Accessors.USER_ROOT_JOB_DIRECTORY, Accessors.GetCompressedDataPath(genomeModel.Seed)
, Accessors.GetRelativeJobDirectory(genomeModel.Seed), Accessors.GetRemoteDownloadLocation(genomeModel.Seed), true, "yr");
continueUpdate = false;
//}
break;
}
// Completed step
case 6:
{
continueUpdate = false;
break;
}
default:
{
// If we have attempted a correction and failed, throw in the towel.
if (outOfRange)
{
throw new IndexOutOfRangeException("The current step of the program is out of bounds after an attempted correction. The current step: "
+ genomeModel.OverallCurrentStep);
}
else
{
outOfRange = true;
// Reset the state to default and have it run through the update method again.
genomeModel.OverallCurrentStep = 1;
genomeModel.OverallStatus = StepDescriptions.GetOverallStepList()[genomeModel.OverallCurrentStep].ToString();
}
break;
}
}
}
}
}
// SSH Connection couldn't be established.
catch (SocketException e)
{
ErrorHandling.error = "The SSH connection couldn't be established. " + e.Message;
}
// The SSH connection was dropped.
catch (SshConnectionException e)
{
ErrorHandling.error = "The connection was terminated unexpectedly. " + e.Message;
}
}
}
public static void SetMasurcaError(SshClient client, GenomeModel genomeModel)
{
genomeModel.MasurcaCurrentStep = -1;
genomeModel.MasurcaStatus = LinuxCommands.GetMasurcaError(client, genomeModel.Seed);
}