public void ParseConfig()
{
Regex sectionHeaderRgx = new Regex(@"^\[.*\]$");
StreamReader inputFile = new StreamReader(Path.GetDirectoryName(System.Reflection.Assembly.GetEntryAssembly().Location) + @"\config.ini");
string line;
//parse config file
while ((line = inputFile.ReadLine()) != null)
{
//skip section headers and empty lines
if (line != @"" && !sectionHeaderRgx.IsMatch(line))
{
string[] fields = line.Split('=');
if (fields.Length == 2)
{
configKeyValuePairs.Add(fields[0], fields[1]);
}
else
{
throw new FileLoadException(@"Line: " + line + @" is malformed");
}
}
}
inputFile.Close();
try
{
//extract parameters
cvxGenSshHostKeyFingerprint = configKeyValuePairs[@"cvxGenSshHostKeyFingerprint"];
cvxGenHostName = configKeyValuePairs[@"cvxGenHostName"];
cvxGenSshUserName = configKeyValuePairs[@"cvxGenSshUserName"];
cvxGenSshPrivateKeyPath = configKeyValuePairs[@"cvxGenSshPrivateKeyPath"];
wotanSshHostKeyFingerprint = configKeyValuePairs[@"wotanSshHostKeyFingerprint"];
wotanHostName = configKeyValuePairs[@"wotanHostName"];
wotanSshUserName = configKeyValuePairs[@"wotanSshUserName"];
wotanSshPrivateKeyPath = configKeyValuePairs[@"wotanSshPrivateKeyPath"];
cvxGenRemoteScriptPath = configKeyValuePairs[@"cvxGenRemoteScriptPath"];
cvxGenRemoteResultsPath = configKeyValuePairs[@"cvxGenRemoteResultsPath"];
cvxGenRemoteArchivePath = configKeyValuePairs[@"cvxGenRemoteArchivePath"];
cvxGenRemoteSAVPath = configKeyValuePairs[@"cvxGenRemoteSAVPath"];
wotanRemoteScriptPath = configKeyValuePairs[@"wotanRemoteScriptPath"];
wotanRemoteResultsPath = configKeyValuePairs[@"wotanRemoteResultsPath"];
wotanRemoteArchivePath = configKeyValuePairs[@"wotanRemoteArchivePath"];
wotanRemoteSAVPath = configKeyValuePairs[@"wotanRemoteSAVPath"];
isMiSeqHost = Convert.ToBoolean(configKeyValuePairs[@"isMiSeqHost"]);
analysisFolderPath = configKeyValuePairs[@"analysisFolderPath"];
outputFolderPath = configKeyValuePairs[@"outputFolderPath"];
deleteOldestLocalRun = Convert.ToBoolean(configKeyValuePairs[@"deleteOldestLocalRun"]);
savExePath = configKeyValuePairs[@"savExePath"];
} catch (KeyNotFoundException e)
{
Framework.WriteLog("Config file missing key-value: " + e.Message, -1);
}
}
public static void CallSequenceAnalysisViewer(string savFilePath, string localRunDir, string savExePath)
{
//write SAV data to table
Process proc = new Process();
proc.StartInfo.FileName = savExePath;
proc.StartInfo.Arguments = localRunDir + ' ' + savFilePath + " -t";
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.RedirectStandardOutput = true;
proc.StartInfo.RedirectStandardError = true;
proc.Start();
Framework.WriteLog(proc.StandardOutput.ReadToEnd(), 0);
Framework.WriteLog(proc.StandardError.ReadToEnd(), 1);
proc.WaitForExit();
proc.Close();
}
private static void onStart()
{
//parse parameters
config.ParseConfig();
Framework.WriteLog("Starting NGSTransfer v" + programVersion, 0);
if (config.getIsMiSeqHost)
{
Framework.WriteLog(@"Determined host to be Illumina MiSeq", 0);
}
else
{
Framework.WriteLog(@"Determined host to be Illumina HiSeq", 0);
}
timer_Tick();
/*pretend infinit loop
* timer_Tick();
*/
}
public static void DirectoryCopyFunction(string sourceDirName, string destDirName, bool copySubDirs)
{
// Get the subdirectories for the specified directory.
DirectoryInfo dir = new DirectoryInfo(sourceDirName);
DirectoryInfo[] dirs = dir.GetDirectories();
if (!dir.Exists)
{
Framework.WriteLog(@"Source directory does not exist or could not be found " + sourceDirName, -1);
throw new DirectoryNotFoundException();
}
// If the destination directory doesn't exist, create it.
if (!Directory.Exists(destDirName))
{
Directory.CreateDirectory(destDirName);
}
// Get the files in the directory and copy them to the new location.
FileInfo[] files = dir.GetFiles();
foreach (FileInfo file in files)
{
string temppath = Path.Combine(destDirName, file.Name);
file.CopyTo(temppath, false);
}
// If copying subdirectories, copy them and their contents to new location.
if (copySubDirs)
{
foreach (DirectoryInfo subdir in dirs)
{
string temppath = Path.Combine(destDirName, subdir.Name);
DirectoryCopyFunction(subdir.FullName, temppath, copySubDirs);
}
}
}
private static void timer_Tick()
{
bool foundCompletedRun = false;
//check if runs are complete but not already uploaded
foreach (string folder in Directory.GetDirectories(config.getAnalysisFolderPath))
{
if (File.Exists(Path.Combine(folder, @"RTAComplete.txt")) && !File.Exists(Path.Combine(folder, @"TransferComplete.txt")))
{
Framework.WriteLog(@"Found job: " + Path.Combine(config.getAnalysisFolderPath, folder), 0);
foundCompletedRun = true;
//transfer data to cluster
try
{
TransferJob job = new TransferJob(folder, config);
job.TransferData();
} catch (Exception e) {
Framework.WriteLog(e.Message, -1);
foundCompletedRun = false;
}
}
}
//delete old runs
if (foundCompletedRun && config.getDeleteOldestLocalRun)
{
//get dir list and sort by date creation
var di = new DirectoryInfo(config.getAnalysisFolderPath);
var directories = di.EnumerateDirectories()
.OrderBy(d => d.CreationTime)
.Select(d => d.Name)
.ToList();
//delete subfolders; protect the last maxSubDirsToKeep (newest) runs
for (int n = 0; n < directories.Count - 5; ++n)
{
if (File.Exists(Path.Combine(directories[n], @"TransferComplete.txt")))
{
try
{
Framework.WriteLog(@"Deleting folder: " + Path.Combine(config.getAnalysisFolderPath, directories[n]), 0);
//Directory.Delete(localRunDir + directories[n], true);
}
catch (Exception e)
{
Framework.WriteLog(@"Could not delete folder: " + e.ToString(), -1);
}
}
}
if (config.getIsMiSeqHost)
{
try
{
Framework.WriteLog(@"Deleting and making folder: " + config.getOutputFolderPath, 0);
//Directory.Delete(config.getOutputFolderPath, true);
//Directory.CreateDirectory(config.getOutputFolderPath);
}
catch (Exception e)
{
Framework.WriteLog(@"Could not delete folder: " + e.ToString(), -1);
}
}
}
}
private static void onStop()
{
Framework.WriteLog("Stoppping NGSTransfer ...", 0);
}
public void TransferData()
{
bool nhsRun;
string[] fields;
SessionOptions sessionOptions = new SessionOptions();
//get run parameters
string runId = Framework.GetRunIdFromRunInfoXml(Path.Combine(localRunDir, @"RunInfo.xml"));
//Parse samplesheet
SampleSheetHeader sampleSheetHeader = new SampleSheetHeader(Path.Combine(localRunDir, @"SampleSheet.csv"));
sampleSheetHeader.Populate();
//write variables to log
Framework.WriteLog(@"Local run directory: " + localRunDir, 0);
Framework.WriteLog(@"Run identifier: " + runId, 0);
Framework.WriteLog(@"Local SampleSheet path: " + Path.Combine(localRunDir, @"SampleSheet.csv"), 0);
Framework.WriteLog(@"Experiment name: " + sampleSheetHeader.getExperimentName, 0);
//determine institution
if (sampleSheetHeader.getInvestigatorName == null || sampleSheetHeader.getInvestigatorName == "" || !sampleSheetHeader.getInvestigatorName.Contains('-'))
{
throw new InvalidDataException(@"Investigator name field not provided or malformed.");
}
//split investigtor name field
fields = sampleSheetHeader.getInvestigatorName.Split('-');
if (fields[1].ToUpper() == @"CU")
{
nhsRun = false;
Framework.WriteLog(@"Investigator name: " + fields[0], 0);
Framework.WriteLog(@"Institution name: Cardiff University", 0);
sessionOptions.SshHostKeyFingerprint = config.getWotanSshHostKeyFingerprint;
sessionOptions.HostName = config.getWotanHostName;
sessionOptions.UserName = config.getWotanSshUserName;
sessionOptions.SshPrivateKeyPath = config.getWotanSshPrivateKeyPath;
}
else if (fields[1].ToUpper() == @"NHS")
{
nhsRun = true;
Framework.WriteLog(@"Investigator name: " + fields[0], 0);
Framework.WriteLog(@"Institution name: NHS", 0);
sessionOptions.SshHostKeyFingerprint = config.getCvxGenSshHostKeyFingerprint;
sessionOptions.HostName = config.getCvxGenHostName;
sessionOptions.UserName = config.getCvxGenSshUserName;
sessionOptions.SshPrivateKeyPath = config.getCvxGenSshPrivateKeyPath;
}
else
{
throw new InvalidDataException(@"Institute not recognised. CU an NHS are only available options.");
}
//convert metrics into table
Framework.WriteLog(@"Writing SAV table ...", 0);
string savFilePath = Path.Combine(Path.GetTempPath(), runId + @"_SAV.txt");
Framework.CallSequenceAnalysisViewer(savFilePath, localRunDir, config.getSavExePath);
//transfer data to cluster
if (nhsRun)
{
Framework.WriteLog(@"Starting transfer to cvx-gen ...", 0);
using (Session session = new Session())
{
string remotePath;
if (config.getIsMiSeqHost)
{
remotePath = config.getCvxGenRemoteArchivePath + @"/miseq/" + runId;
}
else
{
remotePath = config.getCvxGenRemoteArchivePath + @"/hiseq/" + runId;
}
session.Open(sessionOptions);
Framework.WriteLog(@"Connected!", 0);
TransferOperationResult transferResult;
TransferOptions transferOptions = new TransferOptions();
transferOptions.TransferMode = TransferMode.Binary;
//make remote project directory
Framework.WriteLog(@"Creating remote directory: " + remotePath, 0);
session.CreateDirectory(remotePath);
//transfer run folder to archive
Framework.WriteLog(@"Tranfer started ...", 0);
transferResult = session.PutFiles(localRunDir, remotePath, false, transferOptions);
transferResult.Check(); // Throw on any error
//transfer SAV file
transferResult = session.PutFiles(savFilePath, config.getCvxGenRemoteSAVPath, false, transferOptions);
transferResult.Check(); // Throw on any error
//execute remote function
Framework.WriteLog(@"Execuing post-run command ...", 0);
session.ExecuteCommand(@"bash " + config.getCvxGenRemoteScriptPath + ' ' + remotePath + ' ' + config.getCvxGenRemoteResultsPath + '/' + runId + @" >> ~/runPipeline.log");
}
}
else
{
//TODO transfer to Wotan
}
//write run transfer completion
using (StreamWriter file = new StreamWriter(Path.Combine(localRunDir, @"TransferComplete.txt")))
{
file.WriteLine(@"NGSTransferComplete," + Program.programVersion);
file.Close();
}
}