public List<String> GetHeaders(string winSCPPath, string csvDirectory, string csvFile, string rslReport,
string parameters, string windowsOutputPath)
{
List<String> headerList = null;
String reportHeaderFileName = rslReport + "-headers.txt";
using (Session session = new Session())
{
try
{
session.ExecutablePath = winSCPPath;
session.Open(winSCPSessionOptions);
}
catch (Exception e)
{
throw new Exception("Could not open a WinSCP session!\n\n" + e.Message);
}
try
{
DirectoryInfo dir = new DirectoryInfo(windowsOutputPath);
if (!dir.Exists) { dir.Create(); }
}
catch (Exception e)
{
throw new Exception(String.Format("Could not create the following directory: {0}\n\n{1}",
windowsOutputPath, e.Message));
}
String newCsvFileName = rslReport + ".csv";
TransferOptions transferOptions = new TransferOptions();
transferOptions.TransferMode = TransferMode.Binary;
TransferOperationResult transferResult;
try
{
transferResult = session.GetFiles(csvDirectory + csvFile, windowsOutputPath + newCsvFileName, false, transferOptions);
transferResult.Check();
}
catch (Exception e)
{
throw new Exception(String.Format("Could not transfer the file \"{0}\" in Unix server to the Windows output as {1}\n\n{2}",
csvDirectory + csvFile, windowsOutputPath + newCsvFileName, e.Message));
}
try
{
session.ExecuteCommand("rm -f " + csvDirectory + csvFile);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not execute de command: rm {0}\n\n{1}",
csvDirectory + csvFile, e.Message));
}
if (File.Exists(windowsOutputPath + newCsvFileName))
{
headerList = new List<string>();
string line;
try
{
using (var reader = File.OpenText(windowsOutputPath + newCsvFileName))
{
line = reader.ReadLine();
if (line != null)
{
line = line.Replace("\"", "");
headerList = line.Split(',').ToList();
}
}
}
catch (Exception e)
{
throw new Exception(String.Format("Could not open file: {0}\n\n{1}", windowsOutputPath + newCsvFileName, e.Message));
}
try
{
File.Delete(windowsOutputPath + newCsvFileName);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not delete the following file: {0}\n\n{1}",
windowsOutputPath + newCsvFileName, e.Message));
}
}
using (var sw = new StreamWriter(windowsOutputPath + reportHeaderFileName))
{
foreach (string s in headerList) { sw.WriteLine(s); }
}
}
return headerList;
}
public void DownloadCSV(string winSCPPath, string csvDirectory, string serverType, string version,
string csvFile, string report, string portfolio, string windowsOutput)
{
using (Session session = new Session())
{
try
{
// Connect
session.ExecutablePath = winSCPPath;
session.Open(winSCPSessionOptions);
}
catch (Exception e)
{
throw new Exception("Could not open a WinSCP session!\n\n" + e.Message);
}
String rslReport = Path.GetFileNameWithoutExtension(report);
String windowsOutputPath = windowsOutput + @"GFT_DiffTool_Output\"
+ DateTime.Today.ToString("yyyy-MM-dd") + "-" + version + @"\" + serverType + @"\";
try
{
DirectoryInfo dir = new DirectoryInfo(windowsOutputPath);
if (!dir.Exists) { dir.Create(); }
}
catch (Exception e)
{
throw new Exception(String.Format("Could not create the following directory: {0}\n\n{1}",
windowsOutputPath, e.Message));
}
portfolio = portfolio.Replace(' ', '_');
String newCsvFileName = serverType + "." + portfolio + "." + rslReport + "." + version + ".csv";
#region Clean Directory
if (File.Exists(windowsOutputPath + newCsvFileName))
{
try
{
File.Delete(windowsOutputPath + newCsvFileName);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not delete the following file: {0}\n\n{1}",
windowsOutputPath + newCsvFileName, e.Message));
}
}
if (File.Exists(windowsOutputPath + "temp_" + newCsvFileName))
{
try
{
File.Delete(windowsOutputPath + "temp_" + newCsvFileName);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not delete the following file: {0}\n\n{21}",
windowsOutputPath + "temp_" + newCsvFileName, e.Message));
}
}
if (File.Exists(windowsOutputPath + "backup_" + newCsvFileName))
{
try
{
File.Delete(windowsOutputPath + "backup_" + newCsvFileName);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not delete the following file: {0}\n\n{1}",
windowsOutputPath + "backup_" + newCsvFileName, e.Message));
}
}
if (File.Exists(windowsOutputPath + "ERRORS_" + newCsvFileName))
{
try
{
File.Delete(windowsOutputPath + "ERRORS_" + newCsvFileName);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not delete the following file: {0}\n\n{21}",
windowsOutputPath + "ERRORS_" + newCsvFileName, e.Message));
}
}
#endregion
#region Transfer File
TransferOptions transferOptions = new TransferOptions();
transferOptions.TransferMode = TransferMode.Binary;
TransferOperationResult transferResult;
try
{
transferResult = session.GetFiles(csvDirectory + csvFile, windowsOutputPath + newCsvFileName, false, transferOptions);
transferResult.Check();
}
catch (Exception e)
{
throw new Exception(String.Format("Could not transfer the file \"{0}\" in Unix server to the Windows output as {1}\n\n{2}",
csvDirectory + csvFile, windowsOutputPath + newCsvFileName, e.Message));
}
try
{
session.ExecuteCommand("rm -f " + csvDirectory + csvFile);
}
catch (Exception e)
{
throw new Exception(String.Format("Could not execute de command: rm {0}\n\n{1}",
csvDirectory + csvFile, e.Message));
}
#endregion
if (File.Exists(windowsOutputPath + newCsvFileName))
{
string header;
string line = "";
int errorCount = 0;
try
{
using (var reader = File.OpenText(windowsOutputPath + newCsvFileName))
using (var writer = File.CreateText(windowsOutputPath + "temp_" + newCsvFileName))
using (var errorWriter = File.CreateText(windowsOutputPath + "ERRORS_" + newCsvFileName))
{
header = reader.ReadLine();
writer.WriteLine(header);
errorWriter.WriteLine("\"Error #\",\"Description\"");
while (line != null && (line = reader.ReadLine()) != null)
{
if (line.Contains("\"Error #\""))
{
errorCount++;
while ((line = reader.ReadLine()) != null && !header.Equals(line))
{
errorWriter.WriteLine(line);
}
}
else
{
if (!header.Equals(line) && !line.Trim().Equals("") && line.StartsWith("\""))
{
writer.WriteLine(line);
}
}
}
}
File.Replace(windowsOutputPath + "temp_" + newCsvFileName,
windowsOutputPath + newCsvFileName,
windowsOutputPath + "backup_" + newCsvFileName);
if (File.Exists(windowsOutputPath + "backup_" + newCsvFileName))
{
File.Delete(windowsOutputPath + "backup_" + newCsvFileName);
}
if (File.Exists(windowsOutputPath + "ERRORS_" + newCsvFileName) && errorCount == 0)
{
File.Delete(windowsOutputPath + "ERRORS_" + newCsvFileName);
}
}
catch (Exception e)
{
throw new Exception(String.Format("Could not open the file: {0}\n\n{1}", newCsvFileName, e.Message));
}
}
}
}
public void TransferData()
{
bool nhsRun;
string[] fields;
SessionOptions sessionOptions = new SessionOptions();
//get run parameters
string runId = Framework.GetRunIdFromRunInfoXml(Path.Combine(localRunDir, @"RunInfo.xml"));
//Parse samplesheet
SampleSheetHeader sampleSheetHeader = new SampleSheetHeader(Path.Combine(localRunDir, @"SampleSheet.csv"));
sampleSheetHeader.Populate();
//write variables to log
Framework.WriteLog(@"Local run directory: " + localRunDir, 0);
Framework.WriteLog(@"Run identifier: " + runId, 0);
Framework.WriteLog(@"Local SampleSheet path: " + Path.Combine(localRunDir, @"SampleSheet.csv"), 0);
Framework.WriteLog(@"Experiment name: " + sampleSheetHeader.getExperimentName, 0);
//determine institution
if (sampleSheetHeader.getInvestigatorName == null || sampleSheetHeader.getInvestigatorName == "" || !sampleSheetHeader.getInvestigatorName.Contains('-'))
{
throw new InvalidDataException(@"Investigator name field not provided or malformed.");
}
//split investigtor name field
fields = sampleSheetHeader.getInvestigatorName.Split('-');
if (fields[1].ToUpper() == @"CU")
{
nhsRun = false;
Framework.WriteLog(@"Investigator name: " + fields[0], 0);
Framework.WriteLog(@"Institution name: Cardiff University", 0);
sessionOptions.SshHostKeyFingerprint = config.getWotanSshHostKeyFingerprint;
sessionOptions.HostName = config.getWotanHostName;
sessionOptions.UserName = config.getWotanSshUserName;
sessionOptions.SshPrivateKeyPath = config.getWotanSshPrivateKeyPath;
}
else if (fields[1].ToUpper() == @"NHS")
{
nhsRun = true;
Framework.WriteLog(@"Investigator name: " + fields[0], 0);
Framework.WriteLog(@"Institution name: NHS", 0);
sessionOptions.SshHostKeyFingerprint = config.getCvxGenSshHostKeyFingerprint;
sessionOptions.HostName = config.getCvxGenHostName;
sessionOptions.UserName = config.getCvxGenSshUserName;
sessionOptions.SshPrivateKeyPath = config.getCvxGenSshPrivateKeyPath;
}
else
{
throw new InvalidDataException(@"Institute not recognised. CU an NHS are only available options.");
}
//convert metrics into table
Framework.WriteLog(@"Writing SAV table ...", 0);
string savFilePath = Path.Combine(Path.GetTempPath(), runId + @"_SAV.txt");
Framework.CallSequenceAnalysisViewer(savFilePath, localRunDir, config.getSavExePath);
//transfer data to cluster
if (nhsRun)
{
Framework.WriteLog(@"Starting transfer to cvx-gen ...", 0);
using (Session session = new Session())
{
string remotePath;
if (config.getIsMiSeqHost)
{
remotePath = config.getCvxGenRemoteArchivePath + @"/miseq/" + runId;
}
else
{
remotePath = config.getCvxGenRemoteArchivePath + @"/hiseq/" + runId;
}
session.Open(sessionOptions);
Framework.WriteLog(@"Connected!", 0);
TransferOperationResult transferResult;
TransferOptions transferOptions = new TransferOptions();
transferOptions.TransferMode = TransferMode.Binary;
//make remote project directory
Framework.WriteLog(@"Creating remote directory: " + remotePath, 0);
session.CreateDirectory(remotePath);
//transfer run folder to archive
Framework.WriteLog(@"Tranfer started ...", 0);
transferResult = session.PutFiles(localRunDir, remotePath, false, transferOptions);
transferResult.Check(); // Throw on any error
//transfer SAV file
transferResult = session.PutFiles(savFilePath, config.getCvxGenRemoteSAVPath, false, transferOptions);
transferResult.Check(); // Throw on any error
//execute remote function
Framework.WriteLog(@"Execuing post-run command ...", 0);
session.ExecuteCommand(@"bash " + config.getCvxGenRemoteScriptPath + ' ' + remotePath + ' ' + config.getCvxGenRemoteResultsPath + '/' + runId + @" >> ~/runPipeline.log");
}
}
else
{
//TODO transfer to Wotan
}
//write run transfer completion
using (StreamWriter file = new StreamWriter(Path.Combine(localRunDir, @"TransferComplete.txt")))
{
file.WriteLine(@"NGSTransferComplete," + Program.programVersion);
file.Close();
}
}