public List<String> GetHeaders(string winSCPPath, string csvDirectory, string csvFile, string rslReport, string parameters, string windowsOutputPath) { List<String> headerList = null; String reportHeaderFileName = rslReport + "-headers.txt"; using (Session session = new Session()) { try { session.ExecutablePath = winSCPPath; session.Open(winSCPSessionOptions); } catch (Exception e) { throw new Exception("Could not open a WinSCP session!\n\n" + e.Message); } try { DirectoryInfo dir = new DirectoryInfo(windowsOutputPath); if (!dir.Exists) { dir.Create(); } } catch (Exception e) { throw new Exception(String.Format("Could not create the following directory: {0}\n\n{1}", windowsOutputPath, e.Message)); } String newCsvFileName = rslReport + ".csv"; TransferOptions transferOptions = new TransferOptions(); transferOptions.TransferMode = TransferMode.Binary; TransferOperationResult transferResult; try { transferResult = session.GetFiles(csvDirectory + csvFile, windowsOutputPath + newCsvFileName, false, transferOptions); transferResult.Check(); } catch (Exception e) { throw new Exception(String.Format("Could not transfer the file \"{0}\" in Unix server to the Windows output as {1}\n\n{2}", csvDirectory + csvFile, windowsOutputPath + newCsvFileName, e.Message)); } try { session.ExecuteCommand("rm -f " + csvDirectory + csvFile); } catch (Exception e) { throw new Exception(String.Format("Could not execute de command: rm {0}\n\n{1}", csvDirectory + csvFile, e.Message)); } if (File.Exists(windowsOutputPath + newCsvFileName)) { headerList = new List<string>(); string line; try { using (var reader = File.OpenText(windowsOutputPath + newCsvFileName)) { line = reader.ReadLine(); if (line != null) { line = line.Replace("\"", ""); headerList = line.Split(',').ToList(); } } } catch (Exception e) { throw new Exception(String.Format("Could not open file: {0}\n\n{1}", windowsOutputPath + newCsvFileName, e.Message)); } try { File.Delete(windowsOutputPath + newCsvFileName); } catch (Exception e) { throw new Exception(String.Format("Could not delete the following file: {0}\n\n{1}", windowsOutputPath + newCsvFileName, e.Message)); } } using (var sw = new StreamWriter(windowsOutputPath + reportHeaderFileName)) { foreach (string s in headerList) { sw.WriteLine(s); } } } return headerList; }
public void DownloadCSV(string winSCPPath, string csvDirectory, string serverType, string version, string csvFile, string report, string portfolio, string windowsOutput) { using (Session session = new Session()) { try { // Connect session.ExecutablePath = winSCPPath; session.Open(winSCPSessionOptions); } catch (Exception e) { throw new Exception("Could not open a WinSCP session!\n\n" + e.Message); } String rslReport = Path.GetFileNameWithoutExtension(report); String windowsOutputPath = windowsOutput + @"GFT_DiffTool_Output\" + DateTime.Today.ToString("yyyy-MM-dd") + "-" + version + @"\" + serverType + @"\"; try { DirectoryInfo dir = new DirectoryInfo(windowsOutputPath); if (!dir.Exists) { dir.Create(); } } catch (Exception e) { throw new Exception(String.Format("Could not create the following directory: {0}\n\n{1}", windowsOutputPath, e.Message)); } portfolio = portfolio.Replace(' ', '_'); String newCsvFileName = serverType + "." + portfolio + "." + rslReport + "." + version + ".csv"; #region Clean Directory if (File.Exists(windowsOutputPath + newCsvFileName)) { try { File.Delete(windowsOutputPath + newCsvFileName); } catch (Exception e) { throw new Exception(String.Format("Could not delete the following file: {0}\n\n{1}", windowsOutputPath + newCsvFileName, e.Message)); } } if (File.Exists(windowsOutputPath + "temp_" + newCsvFileName)) { try { File.Delete(windowsOutputPath + "temp_" + newCsvFileName); } catch (Exception e) { throw new Exception(String.Format("Could not delete the following file: {0}\n\n{21}", windowsOutputPath + "temp_" + newCsvFileName, e.Message)); } } if (File.Exists(windowsOutputPath + "backup_" + newCsvFileName)) { try { File.Delete(windowsOutputPath + "backup_" + newCsvFileName); } catch (Exception e) { throw new Exception(String.Format("Could not delete the following file: {0}\n\n{1}", windowsOutputPath + "backup_" + newCsvFileName, e.Message)); } } if (File.Exists(windowsOutputPath + "ERRORS_" + newCsvFileName)) { try { File.Delete(windowsOutputPath + "ERRORS_" + newCsvFileName); } catch (Exception e) { throw new Exception(String.Format("Could not delete the following file: {0}\n\n{21}", windowsOutputPath + "ERRORS_" + newCsvFileName, e.Message)); } } #endregion #region Transfer File TransferOptions transferOptions = new TransferOptions(); transferOptions.TransferMode = TransferMode.Binary; TransferOperationResult transferResult; try { transferResult = session.GetFiles(csvDirectory + csvFile, windowsOutputPath + newCsvFileName, false, transferOptions); transferResult.Check(); } catch (Exception e) { throw new Exception(String.Format("Could not transfer the file \"{0}\" in Unix server to the Windows output as {1}\n\n{2}", csvDirectory + csvFile, windowsOutputPath + newCsvFileName, e.Message)); } try { session.ExecuteCommand("rm -f " + csvDirectory + csvFile); } catch (Exception e) { throw new Exception(String.Format("Could not execute de command: rm {0}\n\n{1}", csvDirectory + csvFile, e.Message)); } #endregion if (File.Exists(windowsOutputPath + newCsvFileName)) { string header; string line = ""; int errorCount = 0; try { using (var reader = File.OpenText(windowsOutputPath + newCsvFileName)) using (var writer = File.CreateText(windowsOutputPath + "temp_" + newCsvFileName)) using (var errorWriter = File.CreateText(windowsOutputPath + "ERRORS_" + newCsvFileName)) { header = reader.ReadLine(); writer.WriteLine(header); errorWriter.WriteLine("\"Error #\",\"Description\""); while (line != null && (line = reader.ReadLine()) != null) { if (line.Contains("\"Error #\"")) { errorCount++; while ((line = reader.ReadLine()) != null && !header.Equals(line)) { errorWriter.WriteLine(line); } } else { if (!header.Equals(line) && !line.Trim().Equals("") && line.StartsWith("\"")) { writer.WriteLine(line); } } } } File.Replace(windowsOutputPath + "temp_" + newCsvFileName, windowsOutputPath + newCsvFileName, windowsOutputPath + "backup_" + newCsvFileName); if (File.Exists(windowsOutputPath + "backup_" + newCsvFileName)) { File.Delete(windowsOutputPath + "backup_" + newCsvFileName); } if (File.Exists(windowsOutputPath + "ERRORS_" + newCsvFileName) && errorCount == 0) { File.Delete(windowsOutputPath + "ERRORS_" + newCsvFileName); } } catch (Exception e) { throw new Exception(String.Format("Could not open the file: {0}\n\n{1}", newCsvFileName, e.Message)); } } } }
public void TransferData() { bool nhsRun; string[] fields; SessionOptions sessionOptions = new SessionOptions(); //get run parameters string runId = Framework.GetRunIdFromRunInfoXml(Path.Combine(localRunDir, @"RunInfo.xml")); //Parse samplesheet SampleSheetHeader sampleSheetHeader = new SampleSheetHeader(Path.Combine(localRunDir, @"SampleSheet.csv")); sampleSheetHeader.Populate(); //write variables to log Framework.WriteLog(@"Local run directory: " + localRunDir, 0); Framework.WriteLog(@"Run identifier: " + runId, 0); Framework.WriteLog(@"Local SampleSheet path: " + Path.Combine(localRunDir, @"SampleSheet.csv"), 0); Framework.WriteLog(@"Experiment name: " + sampleSheetHeader.getExperimentName, 0); //determine institution if (sampleSheetHeader.getInvestigatorName == null || sampleSheetHeader.getInvestigatorName == "" || !sampleSheetHeader.getInvestigatorName.Contains('-')) { throw new InvalidDataException(@"Investigator name field not provided or malformed."); } //split investigtor name field fields = sampleSheetHeader.getInvestigatorName.Split('-'); if (fields[1].ToUpper() == @"CU") { nhsRun = false; Framework.WriteLog(@"Investigator name: " + fields[0], 0); Framework.WriteLog(@"Institution name: Cardiff University", 0); sessionOptions.SshHostKeyFingerprint = config.getWotanSshHostKeyFingerprint; sessionOptions.HostName = config.getWotanHostName; sessionOptions.UserName = config.getWotanSshUserName; sessionOptions.SshPrivateKeyPath = config.getWotanSshPrivateKeyPath; } else if (fields[1].ToUpper() == @"NHS") { nhsRun = true; Framework.WriteLog(@"Investigator name: " + fields[0], 0); Framework.WriteLog(@"Institution name: NHS", 0); sessionOptions.SshHostKeyFingerprint = config.getCvxGenSshHostKeyFingerprint; sessionOptions.HostName = config.getCvxGenHostName; sessionOptions.UserName = config.getCvxGenSshUserName; sessionOptions.SshPrivateKeyPath = config.getCvxGenSshPrivateKeyPath; } else { throw new InvalidDataException(@"Institute not recognised. CU an NHS are only available options."); } //convert metrics into table Framework.WriteLog(@"Writing SAV table ...", 0); string savFilePath = Path.Combine(Path.GetTempPath(), runId + @"_SAV.txt"); Framework.CallSequenceAnalysisViewer(savFilePath, localRunDir, config.getSavExePath); //transfer data to cluster if (nhsRun) { Framework.WriteLog(@"Starting transfer to cvx-gen ...", 0); using (Session session = new Session()) { string remotePath; if (config.getIsMiSeqHost) { remotePath = config.getCvxGenRemoteArchivePath + @"/miseq/" + runId; } else { remotePath = config.getCvxGenRemoteArchivePath + @"/hiseq/" + runId; } session.Open(sessionOptions); Framework.WriteLog(@"Connected!", 0); TransferOperationResult transferResult; TransferOptions transferOptions = new TransferOptions(); transferOptions.TransferMode = TransferMode.Binary; //make remote project directory Framework.WriteLog(@"Creating remote directory: " + remotePath, 0); session.CreateDirectory(remotePath); //transfer run folder to archive Framework.WriteLog(@"Tranfer started ...", 0); transferResult = session.PutFiles(localRunDir, remotePath, false, transferOptions); transferResult.Check(); // Throw on any error //transfer SAV file transferResult = session.PutFiles(savFilePath, config.getCvxGenRemoteSAVPath, false, transferOptions); transferResult.Check(); // Throw on any error //execute remote function Framework.WriteLog(@"Execuing post-run command ...", 0); session.ExecuteCommand(@"bash " + config.getCvxGenRemoteScriptPath + ' ' + remotePath + ' ' + config.getCvxGenRemoteResultsPath + '/' + runId + @" >> ~/runPipeline.log"); } } else { //TODO transfer to Wotan } //write run transfer completion using (StreamWriter file = new StreamWriter(Path.Combine(localRunDir, @"TransferComplete.txt"))) { file.WriteLine(@"NGSTransferComplete," + Program.programVersion); file.Close(); } }