private void btnRun_Click(object sender, EventArgs e) { DataTableResponseMessage dtRespMsg = null; try { templateDataGridView.DataSource = null; ClearMessage(); UserMessage("Running"); ProcessorDTO proc = comboBox1.Items[comboBox1.SelectedIndex] as ProcessorDTO; txtID.Text = proc.UniqueId; txtName.Text = proc.Name; txtDesc.Text = proc.Description; txtFileType.Text = proc.InstrumentFileType; ProcessorManager procMgr = new ProcessorManager(); //string output = @"E:\lims\LIMSDesktop\bin\Debug\netcoreapp3.0\Processors\Output\file.csv"; //string procPaths = @"E:\lims\lims_server\app_files\processors"; dtRespMsg = procMgr.ExecuteProcessor(proc.Path, txtName.Text, txtInput.Text); GC.Collect(); GC.WaitForPendingFinalizers(); if (dtRespMsg == null) { //dtRespMsg = new DataTableResponseMessage(); UserMessage(string.Format("Error processing file {0} with processor {1}", txtInput.Text, txtName.Text)); //dtRespMsg.ErrorMessage = string.Format("Error processing file {0} with processor {1}", txtInput.Text, txtName.Text); } if (dtRespMsg.ErrorMessage != null) { LogMessage(dtRespMsg.ErrorMessage); } if (!string.IsNullOrWhiteSpace(dtRespMsg.LogMessage)) { LogMessage(dtRespMsg.LogMessage); } if (dtRespMsg.TemplateData != null) { templateDataGridView.DataSource = dtRespMsg.TemplateData; UserMessage("Success"); } } catch (Exception ex) { LogMessage(string.Format("Error executing processor {0} with input {1}", txtID.Text, txtInput.Text)); LogMessage($"Error: {ex.Message}"); if (dtRespMsg != null && dtRespMsg.LogMessage != null) { LogMessage(dtRespMsg.LogMessage); } } }
//KW //May 13, 2021 //Changed on request by Curtis Callahan via email on April 06, 2021 //private readonly string analyteID = "dsDNA"; //KW: July 20, 2021 //Changed on request by Curtis Callahan via email on July 20, 2021 //Analyte ID is now in cell I1. //private readonly string analyteID = "dsDNA HS"; public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = null; try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); DataTable dt = GetDataTable(); FileInfo fi = new FileInfo(input_file); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; //This is a new way of using the 'using' keyword with braces using var package = new ExcelPackage(fi); //Data is in the 1st sheet var worksheet = package.Workbook.Worksheets[0]; //Worksheets are zero-based index string name = worksheet.Name; int startRow = worksheet.Dimension.Start.Row; int startCol = worksheet.Dimension.Start.Column; int numRows = worksheet.Dimension.End.Row; int numCols = worksheet.Dimension.End.Column; string analyteID = GetXLStringValue(worksheet.Cells[1, 9]); if (string.IsNullOrWhiteSpace(analyteID)) { string msg = string.Format("Input file is not in correct format. Row 1, Column 9 should contain an analyte ID", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } string wellID = GetXLStringValue(worksheet.Cells[18, 1]); if (!"Well ID".Equals(wellID, StringComparison.OrdinalIgnoreCase)) { string msg = string.Format("Input file is not in correct format. Row 18, Column 1 should contain value 'Well ID'. {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } // Row 18 starts the data //Data File|| Well ID Name Well 485/20,528/20 [Concentration] //Template || Aliquot Description Measured Value //Analyte Identifier is dsDNA for every record for (int row = 18; row <= numRows; row++) { wellID = GetXLStringValue(worksheet.Cells[row, 1]); //if the cell is empty then start new data block if (string.IsNullOrWhiteSpace(wellID)) { continue; } //if the cell contains 'Well ID' then start new data block if (wellID.Equals("Well ID", StringComparison.OrdinalIgnoreCase)) { continue; } string[] aliquot_dilFactor = GetAliquotDilutionFactor(GetXLStringValue(worksheet.Cells[row, 2])); string aliquot = aliquot_dilFactor[0]; string description = GetXLStringValue(worksheet.Cells[row, 3]); string measuredVal = GetXLStringValue(worksheet.Cells[row, 5]); //string dilFactor = aliquot_dilFactor[1]; //KW June 9, 2021 //This change implemented at the request of Curtis Callahan string dilFactor = GetXLStringValue(worksheet.Cells[row, 6]); //If measured value is “<0.0” report value as 0 //If measured value is “>1050.0” report value as 1050 if (measuredVal.Contains("<")) { measuredVal = "0"; } else if (measuredVal.Contains(">")) { measuredVal = "1050"; } DataRow dr = dt.NewRow(); dr["Aliquot"] = aliquot; if (!string.IsNullOrWhiteSpace(dilFactor)) { dr["Dilution Factor"] = dilFactor; } else { dr["Dilution Factor"] = 1; } dr["Description"] = description; dr["Measured Value"] = measuredVal; dr["Analyte Identifier"] = analyteID; dt.Rows.Add(dr); } rm.TemplateData = dt.Copy(); } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = null; try { //Using ExcelDataReader Package rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); FileInfo fi = new FileInfo(input_file); DataTableCollection tables; System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); using (var stream = File.Open(input_file, FileMode.Open, FileAccess.Read)) { using (var reader = ExcelReaderFactory.CreateReader(stream)) { var result = reader.AsDataSet(); tables = result.Tables; } } DataTable dt_template = GetDataTable(); dt_template.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); TemplateField[] fields = Fields; var worksheet = tables[0]; int numRows = worksheet.Rows.Count; int numCols = worksheet.Columns.Count; for (int row = 1; row < numRows; row++) { string aliquot_id = worksheet.Rows[row][0].ToString(); DateTime analysis_datetime = fi.CreationTime.Date.Add(DateTime.Parse(worksheet.Rows[row][8].ToString()).TimeOfDay); double measured_val = Convert.ToDouble(worksheet.Rows[row][1].ToString()); string analyte_id = "NH3"; double dilution_factor = Convert.ToDouble(worksheet.Rows[row][3].ToString()); string comment = worksheet.Rows[row][4].ToString(); DataRow dr = dt_template.NewRow(); dr[0] = aliquot_id; dr[1] = analyte_id; dr[2] = measured_val; dr[4] = dilution_factor; dr[5] = analysis_datetime; dr[6] = comment; dt_template.Rows.Add(dr); } rm.TemplateData = dt_template; } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = new DataTableResponseMessage(); try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); FileInfo fi = new FileInfo(input_file); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; using var package = new ExcelPackage(fi); //Data is in the 2nd sheet var worksheet = package.Workbook.Worksheets[1]; //Worksheets are zero-based index string name = worksheet.Name; int startRow = worksheet.Dimension.Start.Row; int startCol = worksheet.Dimension.Start.Column; int numRows = worksheet.Dimension.End.Row; int numCols = worksheet.Dimension.End.Column; DataTable dt_template = GetDataTable(); dt_template.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); TemplateField[] fields = Fields; //The columns in the data file are as follows through column J // A B C D E F G H I J //SED_ID Name Date/Time Assay Conc. Units Stock Conc. Units Assay Type Sample Vol (µL) Dilution Factor //^^^^^^^ ^^^^^ ^^^^^^^^ ^^^^^^^^^ ^^^^^^^^^^^^^^^ //Aliquot AssayType Analysis Measured Dilution Factor //ID Date/Time Value for (int row = 2; row <= numRows; row++) { string aliquot_id = GetXLStringValue(worksheet.Cells[row, 1]); DateTime analysis_datetime = GetXLDateTimeValue(worksheet.Cells[row, 3]); double measured_val = default; ExcelRange rng_meas_val = worksheet.Cells[row, 4]; if (rng_meas_val != null && rng_meas_val.Value != null) { string msr_val = rng_meas_val.Value.ToString().Trim(); if (string.Compare(msr_val, "<0.50") == 0) { measured_val = default; } else { measured_val = GetXLDoubleValue(worksheet.Cells[row, 4]); } } string analyte_id = GetXLStringValue(worksheet.Cells[row, 8]); double dilution_factor = GetXLDoubleValue(worksheet.Cells[row, 10]); DataRow dr = dt_template.NewRow(); dr[0] = aliquot_id; dr[5] = analysis_datetime; dr[2] = measured_val; dr[1] = analyte_id; dr[4] = dilution_factor; dt_template.Rows.Add(dr); } rm.TemplateData = dt_template; } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = new DataTableResponseMessage(); DataTable dt = GetDataTable(); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(input_file); string aliquot = ""; string dilutionFactor = ""; DateTime analysisDateTime = DateTime.MinValue; try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); using (StreamReader sr = new StreamReader(input_file)) { int idxRow = 1; string line; while ((line = sr.ReadLine()) != null) { line = line.Trim(); idxRow++; //Data starts on line 21 if (idxRow < 21) { continue; } //Should not be any more blank lines until we get to the end of the data if (string.IsNullOrWhiteSpace(line)) { break; } //The sheet can contain mulitple data sets //The first row of each data set block contains the LIMS ID and analysis date //e.g. SW846_01DEC_18-2,AW325-S-38-01,,,,,,01-Dec-18,13:34:02 // ^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^ // aliquot analysis date time //The data looks like: //e.g. 1,PFOA,30,7.0112,0,214.703,4589,16.0753,dd,2.6196,0,412.9 > 369,,20,26712.1,607300,7.0112,,,,,0,7.0112,0,0,0,6.9624,7.0491,214.703,0,412.9 > 169,1,4.5385,1,47.307,1035,3.169,556.637,108.689,1e-012,1e-012,0,, // ^^^^ ^^^^^^^ ^^^^^^ // analyte id Area Measured conc string[] tokens = line.Split(','); //If this is an int then its data, otherwise start of new dataset string col1 = tokens[0]; int id; if (!Int32.TryParse(col1, out id)) { //string[] aliquot_dilFactor = GetAliquotDilutionFactor(tokens[1]); aliquot = tokens[1]; //dilutionFactor = aliquot_dilFactor[1]; string date = tokens[tokens.Length - 2] + " " + tokens[tokens.Length - 1]; analysisDateTime = Convert.ToDateTime(date); continue; } DataRow dr = dt.NewRow(); //Aliquot dr[0] = aliquot; //Analyte id dr[1] = tokens[1]; //Measured value if (string.IsNullOrWhiteSpace(tokens[9])) { dr[2] = 0.0; } else { dr[2] = tokens[9]; } //Dilution factor //dr[4] = dilutionFactor; //Date/time dr[5] = analysisDateTime; //User defined 1 if (string.IsNullOrWhiteSpace(tokens[5])) { dr[8] = 0.0; } else { dr[8] = tokens[5]; } dt.Rows.Add(dr); } rm.TemplateData = dt; } } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public async System.Threading.Tasks.Task RunTask(string id) { var task = await _context.Tasks.SingleAsync(t => t.id == id); Log.Information("Executing Task. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID); // Step 1: If status!="SCHEDULED" cancel task if (!task.status.Equals("SCHEDULED")) { Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Current Status: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.status, "Task status is not marked as schedulled."); await this.UpdateStatus(task.id, "CANCELLED", "Task status was set to: " + task.status); return; } // Step 2: Change status to "STARTING" await this.UpdateStatus(task.id, "STARTING", ""); var workflow = await _context.Workflows.Where(w => w.id == task.workflowID).FirstOrDefaultAsync(); if (workflow == null) { Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Unable to find Workflow for the Task."); await this.UpdateStatus(task.id, "CANCELLED", "Error attempting to get workflow of this task. Workflow ID: " + task.workflowID); return; } // Step 3: Check source directory for files List <string> files = new List <string>(); string dirFileMessage = ""; if (new DirectoryInfo(@workflow.inputFolder).Exists) { files = Directory.GetFiles(@workflow.inputFolder).ToList(); } else { dirFileMessage = String.Format("Input directory {0} not found", workflow.inputFolder); Log.Information(dirFileMessage); } // Step 4: If directory or files do not exist reschedule task if (files.Count == 0) { dirFileMessage = (dirFileMessage.Length > 0) ? dirFileMessage : String.Format("No files found in directory: {0}", workflow.inputFolder); await this.UpdateStatus(task.id, "SCHEDULED", dirFileMessage); var newSchedule = new Hangfire.States.ScheduledState(TimeSpan.FromMinutes(workflow.interval)); task.start = DateTime.Now.AddMinutes(workflow.interval); await _context.SaveChangesAsync(); try { BackgroundJobClient backgroundClient = new BackgroundJobClient(); backgroundClient.ChangeState(task.taskID, newSchedule); Log.Information("Task Rescheduled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input Directory: {3}, Message: {4}", task.workflowID, task.id, task.taskID, workflow.inputFolder, "No files found in input directory."); } catch (Exception) { Log.Warning("Error attempting to reschedule Hangfire job. Workflow ID: {0}, task ID: {1}", task.workflowID, task.id); } return; } // Step 5: If file does exist, update task inputFile then compare against previous Tasks. task.inputFile = files.First(); task.status = "PROCESSING"; await _context.SaveChangesAsync(); bool alreadyCompleted = await this.InputFileCheck(task.inputFile, task.workflowID); if (alreadyCompleted) { Log.Information("Hash input file match for WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Input File: {3}, Message: {4}", task.workflowID, task.id, task.taskID, task.inputFile, "Rerunning task after removing file."); try { File.Delete(task.inputFile); Log.Information("Hash input file match successfully deleted. WorkflowID: {0}, ID: {1}, Input File: {2}", task.workflowID, task.id, task.inputFile); } catch (FileNotFoundException ex) { Log.Warning("Error unable to delete input file after hash file match with previous input file. Workflow ID: {0}, ID: {1}", task.workflowID, task.id); } string statusMessage = String.Format("Input file: {0} matches previously processed input file", task.inputFile); await this.UpdateStatus(task.id, "SCHEDULED", statusMessage); await this.RunTask(task.id); return; } ProcessorManager pm = new ProcessorManager(); string config = "./app_files/processors"; ProcessorDTO processor = pm.GetProcessors(config).Find(p => p.Name.ToLower() == workflow.processor.ToLower()); if (processor == null) { Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}, Processor: {4}", task.workflowID, task.id, task.taskID, "Unable to find Processor for the Task.", workflow.processor); await this.UpdateStatus(task.id, "CANCELLED", "Error, invalid processor name. Name: " + workflow.processor); return; } try { // Step 6: Run processor on source file if (!new DirectoryInfo(@workflow.outputFolder).Exists) { Directory.CreateDirectory(@workflow.outputFolder); } } catch (UnauthorizedAccessException ex) { Log.Warning("Task unable to create output directory, unauthorized access exception. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Output Directory: {3}", task.workflowID, task.id, task.taskID, workflow.outputFolder); } Dictionary <string, ResponseMessage> outputs = new Dictionary <string, ResponseMessage>(); string file = task.inputFile; DataTableResponseMessage result = pm.ExecuteProcessor(processor.Path, processor.Name, file); GC.Collect(); GC.WaitForPendingFinalizers(); if (result.ErrorMessage == null && result.TemplateData != null) { var output = pm.WriteTemplateOutputFile(workflow.outputFolder, result.TemplateData); outputs.Add(file, output); } else { string errorMessage = ""; string logMessage = ""; if (result.TemplateData == null) { errorMessage = "Processor results template data is null. "; } if (result.ErrorMessage != null) { errorMessage = errorMessage + result.ErrorMessage; logMessage = errorMessage; } if (result.LogMessage != null) { logMessage = result.LogMessage; } await this.UpdateStatus(task.id, "CANCELLED", "Error processing data: " + errorMessage); Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, logMessage); return; } // Step 7: Check if output file exists bool processed = false; for (int i = 0; i < outputs.Count; i++) { var output = outputs.ElementAt(i); string outputPath = output.Value.OutputFile; // Step 8: If output file does exist, update task outputFile and delete input file if (File.Exists(outputPath)) { processed = true; string fileName = System.IO.Path.GetFileName(output.Key); string inputPath = System.IO.Path.Combine(workflow.inputFolder, fileName); DataBackup dbBackup = new DataBackup(); dbBackup.DumpData(id, inputPath, outputPath); try { File.Delete(inputPath); } catch (Exception ex) { Log.Warning("Error unable to delete input file after successful processing. Workflow ID: {0}, ID: {1}", task.workflowID, task.id); } task.outputFile = outputPath; await _context.SaveChangesAsync(); } else { await this.UpdateStatus(task.id, "SCHEDULED", "Error unable to export output. Error Messages: " + output.Value.ErrorMessage); } } // Step 9: Change task status to COMPLETED // Step 10: Create new Task and schedule string newStatus = ""; if (processed) { newStatus = "COMPLETED"; Log.Information("Task Completed. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}", task.workflowID, task.id, task.taskID); try { if (files.Count > 1) { await this.CreateNewTask(workflow.id, 0); } else { await this.CreateNewTask(workflow.id, workflow.interval); } } catch (Exception) { Log.Warning("Error creating new Hangfire job after successful job. Workflow ID: {0}, ID: {1}", task.workflowID, task.id); } } else { newStatus = "CANCELLED"; Log.Information("Task Cancelled. WorkflowID: {0}, ID: {1}, Hangfire ID: {2}, Message: {3}", task.workflowID, task.id, task.taskID, "Failed to process input file."); } await this.UpdateStatus(task.id, newStatus); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = new DataTableResponseMessage(); try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); DataTable dt = GetDataTable(); FileInfo fi = new FileInfo(input_file); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; //This is a new way of using the 'using' keyword with braces using var package = new ExcelPackage(fi); var worksheet = package.Workbook.Worksheets[1]; //Worksheets are zero-based index string name = worksheet.Name; //File validation if (worksheet.Dimension == null) { string msg = string.Format("No data in Sheet 1 in InputFile: {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } int startRow = worksheet.Dimension.Start.Row; int startCol = worksheet.Dimension.Start.Column; int numRows = worksheet.Dimension.End.Row; int numCols = worksheet.Dimension.End.Column; for (int col = 3; col <= numCols; col++) { string aliquot = GetXLStringValue(worksheet.Cells[1, col]); if (aliquot.ToLower() == "reads/otu") { break; } if (string.IsNullOrWhiteSpace(aliquot)) { break; } for (int row = 2; row <= numRows; row++) { DataRow dr = dt.NewRow(); string analyteID = GetXLStringValue(worksheet.Cells[row, 1]); string desc = GetXLStringValue(worksheet.Cells[row, 2]); string measuredVal = GetXLStringValue(worksheet.Cells[row, col]); if (string.IsNullOrWhiteSpace(measuredVal)) { measuredVal = "0"; } dr["Aliquot"] = aliquot; dr["Analyte Identifier"] = analyteID; dr["Measured Value"] = measuredVal; dr["Description"] = desc; dt.Rows.Add(dr); } } rm.TemplateData = dt; } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = null; try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); DataTable dt = GetDataTable(); List <AliquotAnalyte> lstAliquotAnalytes = new List <AliquotAnalyte>(); FileInfo fi = new FileInfo(input_file); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; //This is a new way of using the 'using' keyword with braces using var package = new ExcelPackage(fi); //Start Sheet 2 ------------------------------------------------------------------------------- //Data is in the 2nd sheet var worksheet2 = package.Workbook.Worksheets[1]; //Worksheets are zero-based index string name = worksheet2.Name; //File validation if (worksheet2.Dimension == null) { string msg = string.Format("No data in Sheet2 in InputFile: {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } int startRow = worksheet2.Dimension.Start.Row; int startCol = worksheet2.Dimension.Start.Column; int numRows = worksheet2.Dimension.End.Row; int numCols = worksheet2.Dimension.End.Column; //More file validation string sval = GetXLStringValue(worksheet2.Cells[8, 1]); if (string.Compare(sval, "Data File", true) != 0) { string msg = string.Format("Input file is not in correct format. String 'Data File' missing from cell A8. {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } List <string> lstAnalyteIDs = new List <string>(); //Sheet 2, Row 8, starting at column 2 contains Aliquots for (int col = 2; col <= numCols; col++) { string sCell = GetXLStringValue(worksheet2.Cells[8, col]); if (!"All Flags".Equals(sCell, StringComparison.OrdinalIgnoreCase)) { lstAnalyteIDs.Add(sCell); } else { break; } } //KW: July 19, 2021 //"Sample Acquisition Date" - was assumed to be in last column //We will search for it instead string analyzeDate = ""; int analyzeDateColNum = -1; for (int i = numCols; i > 1; i--) { analyzeDate = GetXLStringValue(worksheet2.Cells[8, i]); if (analyzeDate.Equals("Sample Acquisition Date", StringComparison.OrdinalIgnoreCase)) { analyzeDateColNum = i; break; } } if (analyzeDateColNum < 0) { string msg = "Sample Acquisition Date not in right column: Row {0}, Column {1}. File: {2}"; rm.LogMessage = string.Format(msg, 8, numCols, input_file); rm.ErrorMessage = string.Format(msg, 8, numCols, input_file); return(rm); } //string analyzeDate = GetXLStringValue(worksheet2.Cells[8, numCols]); //if (!analyzeDate.Equals("Sample Acquisition Date", StringComparison.OrdinalIgnoreCase)) //{ // string msg = "Sample Acquisition Date not in right column: Row {0}, Column {1}. File: {2}"; // rm.LogMessage = string.Format(msg, 8, numCols, input_file); // rm.ErrorMessage = string.Format(msg, 8, numCols, input_file); // return rm; //} int numAnalytes = lstAnalyteIDs.Count; //Sheet 2, Row 9 down, Column 1 contains Aliquot name //Sheet 2, Row 9 down, Column 2 until 'All Flags' contain data for (int row = 9; row <= numRows; row++) { //Sheet 2, Row 9 down, Column 1 contains Aliquot name string aliquot = GetXLStringValue(worksheet2.Cells[row, 1]); analyzeDate = GetXLStringValue(worksheet2.Cells[row, analyzeDateColNum]); for (int col = 2; col < numAnalytes; col++) { DataRow dr = dt.NewRow(); //dr["Aliquot"] = aliquot; //dr["Analyte Identifier"] = lstAnalyteIDs[col - 2]; string analyteID = lstAnalyteIDs[col - 2]; //dr["Analysis Date/Time"] = analyzeDate; //dr["Measured Value"] = GetXLStringValue(worksheet2.Cells[row, col]); string measuredVal = GetXLStringValue(worksheet2.Cells[row, col]); //dt.Rows.Add(dr); AliquotAnalyte al = new AliquotAnalyte(aliquot, analyteID, measuredVal, analyzeDate); lstAliquotAnalytes.Add(al); } } //End Sheet 2 ------------------------------------------------------------------------------- //Start Sheet 4 ----------------------------------------------------------------------------- var worksheet4 = package.Workbook.Worksheets[3]; //Worksheets are zero-based index name = worksheet4.Name; //File validation if (worksheet4.Dimension == null) { string msg = string.Format("No data in Sheet4 in InputFile: {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } startRow = worksheet4.Dimension.Start.Row; startCol = worksheet4.Dimension.Start.Column; numRows = worksheet4.Dimension.End.Row; numCols = worksheet4.Dimension.End.Column; lstAnalyteIDs = new List <string>(); //Sheet 4, Row 8, starting at column 2 contains Aliquots for (int col = 2; col <= numCols; col++) { string sCell = GetXLStringValue(worksheet4.Cells[8, col]); if (!"All Flags".Equals(sCell, StringComparison.OrdinalIgnoreCase)) { lstAnalyteIDs.Add(sCell); } else { break; } } numAnalytes = lstAnalyteIDs.Count; //Sheet 4, Row 9 down, Column 1 contains Aliquot name //Sheet 4, Row 9 down, Column 2 until 'All Flags' contain data for (int row = 9; row <= numRows; row++) { //Sheet 2, Row 9 down, Column 1 contains Aliquot name string aliquot = GetXLStringValue(worksheet4.Cells[row, 1]); for (int col = 2; col < numAnalytes; col++) { string analyte = lstAnalyteIDs[col - 2]; //string[] keys = new string[2]; //keys[0] = aliquot; //keys[1] = analyte; var al = lstAliquotAnalytes.Find(x => x.Aliquot.Equals(aliquot, StringComparison.OrdinalIgnoreCase) && x.AnalyteID.Equals(analyte, StringComparison.OrdinalIgnoreCase)); //DataRow drow = dt.Rows.Find(keys); //DataRow[] rows = dt.Select(qry); if (al == null) { //DataRow dr = dt.NewRow(); //dr["Aliquot"] = aliquot; //dr["Analyte Identifier"] = analyte; //dr["User Defined 1"] = GetXLStringValue(worksheet4.Cells[row, col]); string userDefined1 = GetXLStringValue(worksheet4.Cells[row, col]); AliquotAnalyte al2 = new AliquotAnalyte(aliquot, analyte, "", "", userDefined1); lstAliquotAnalytes.Add(al2); } else { //drow["User Defined 1"] = GetXLStringValue(worksheet4.Cells[row, col]); al.UserDefined1 = GetXLStringValue(worksheet4.Cells[row, col]); } } } for (int i = 0; i < lstAliquotAnalytes.Count; i++) { DataRow dr = dt.NewRow(); dr["Aliquot"] = lstAliquotAnalytes[i].Aliquot; dr["Analyte Identifier"] = lstAliquotAnalytes[i].AnalyteID; dr["Analysis Date/Time"] = lstAliquotAnalytes[i].AnalysisDateTime; if (!string.IsNullOrWhiteSpace(lstAliquotAnalytes[i].MeasuredValue)) { dr["Measured Value"] = lstAliquotAnalytes[i].MeasuredValue; } dr["User Defined 1"] = lstAliquotAnalytes[i].UserDefined1; dt.Rows.Add(dr); } rm.TemplateData = dt; } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = null; try { //Using ExcelDataReader Package rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); FileInfo fi = new FileInfo(input_file); DataTableCollection tables; System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); using (var stream = File.Open(input_file, FileMode.Open, FileAccess.Read)) { using (var reader = ExcelReaderFactory.CreateReader(stream)) { var result = reader.AsDataSet(); tables = result.Tables; } } DataTable dt_template = GetDataTable(); dt_template.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); TemplateField[] fields = Fields; int numTables = tables.Count; for (int i = 0; i < numTables; i++) { var worksheet = tables[i]; string analyte_id = worksheet.Rows[2][0].ToString(); int numRows = worksheet.Rows.Count; int numCols = worksheet.Columns.Count; //Data row starts on Excel row 6, 5 for zero based for (int row = 5; row < numRows; row++) { string fileName = worksheet.Rows[row][0].ToString().Trim(); if (String.IsNullOrWhiteSpace(fileName)) { break; } string aliquot_id = worksheet.Rows[row][2].ToString(); //Measured val can contain string of 'NA' or 'NF' string measured_val = worksheet.Rows[row][5].ToString().Trim(); double d_measured_val; if (!Double.TryParse(measured_val, out d_measured_val)) { d_measured_val = 0.0; } double dilution_factor = Convert.ToDouble(worksheet.Rows[row][40].ToString()); DateTime analysis_datetime = fi.CreationTime.Date.Add(DateTime.Parse(worksheet.Rows[row][30].ToString()).TimeOfDay); //Area string userDefined1 = worksheet.Rows[row][14].ToString(); //ISTD Area string userDefined2 = worksheet.Rows[row][16].ToString(); DataRow dr = dt_template.NewRow(); dr[0] = aliquot_id; dr[1] = analyte_id; dr[2] = d_measured_val; dr[4] = dilution_factor; dr[5] = analysis_datetime; dr[8] = userDefined1; dr[9] = userDefined2; dt_template.Rows.Add(dr); } } rm.TemplateData = dt_template; } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = new DataTableResponseMessage(); DataTable dt = GetDataTable(); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(input_file); DateTime analysisDateTime = DateTime.MinValue; //Data looks like this //Sample ID,A/S Loc,User Name 2,User Value 2,User Name 1,User Value 1,Sample Units,Aliquot Vol.,Diluted To Vol.,Analyst Name,Result Name,User Name 3,User Value 3,Analyte Name,Reported Conc (Samp),QC Recovery,Repl No1,Conc (Samp)1,Date1,Time1,Repl No2,Conc (Samp)2,Date2,Time2,Repl No3,Conc (Samp)3,Date3,Time3 //ICV,5, ,4 / 1 / 2021, , , , , , ,STAD_7753_RW_051021, , ,Ag 328.068,4.96,99.39916292,1,4.87881223,5 / 10 / 2021,2:45:46 PM,2,4.971885733,5 / 10 / 2021,2:45:52 PM,3,5.029356726,5 / 10 / 2021,2:45:58 PM try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); using (StreamReader sr = new StreamReader(input_file)) { int idxRow = 0; string line; while ((line = sr.ReadLine()) != null) { //First row is header. We'll skip it if (idxRow < 1) { idxRow++; continue; } line = line.Trim(); if (string.IsNullOrWhiteSpace(line)) { break; } string[] tokens = line.Split(','); string aliquot = tokens[0].Trim(); //Handle case of empty lines at end of file if (string.IsNullOrWhiteSpace(aliquot)) { break; } //Dilution factor string dilFactor = tokens[12]; //Column M - User value 3 int dilutionFactor; if (!int.TryParse(dilFactor, out dilutionFactor)) { dilutionFactor = 1; } //Column N - Analyte Name //Need to split this column components - Analyte ID and User Defined 1 - split on white space string analyteName = tokens[13]; string[] analyteNameTokens = analyteName.Split(' '); string analyteID = analyteNameTokens[0].Trim(); string userDefined1 = analyteNameTokens[1].Trim(); double measuredVal; if (!double.TryParse(tokens[14], out measuredVal)) { measuredVal = 0.0; } string date = tokens[26].Trim(); string time = tokens[27].Trim(); DateTime date_time = DateTime.Parse(date + " " + time); string dateTime = date_time.ToString("MM/dd/yy hh:mm tt"); DataRow dr = dt.NewRow(); dr["Aliquot"] = aliquot; dr["Analyte Identifier"] = analyteID; dr["Measured Value"] = measuredVal; dr["Analysis Date/Time"] = dateTime; dr["Dilution Factor"] = dilutionFactor; dr["User Defined 1"] = userDefined1; dt.Rows.Add(dr); idxRow++; } rm.TemplateData = dt; } } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = null; try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); DataTable dt = GetDataTable(); FileInfo fi = new FileInfo(input_file); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; //This is a new way of using the 'using' keyword with braces using var package = new ExcelPackage(fi); //Data is in the 1st sheet var worksheet = package.Workbook.Worksheets[0]; //Worksheets are zero-based index string name = worksheet.Name; int startRow = worksheet.Dimension.Start.Row; int startCol = worksheet.Dimension.Start.Column; int numRows = worksheet.Dimension.End.Row; int numCols = worksheet.Dimension.End.Column; string analyte_post_acidification = "post-acidification"; //Rows and columns start at 1 not 0 //First row is header data for (int row = 2; row < numRows; row++) { string aliquot = GetXLStringValue(worksheet.Cells[row, 2]); //Lets check for empty cell, assume we are done if we hit one if (string.IsNullOrWhiteSpace(aliquot)) { break; } string description = GetXLStringValue(worksheet.Cells[row, 11]); double dilution_factor = GetXLDoubleValue(worksheet.Cells[row, 13]); double user_defined1 = GetXLDoubleValue(worksheet.Cells[row, 14]); DateTime analysis_date = GetXLDateTimeValue(worksheet.Cells[row, 26]); //Columns O-W contain measured values //Analyte ID is based on value in column L (0 or 1); int acidified = Convert.ToInt32(GetXLDoubleValue(worksheet.Cells[row, 12])); for (int col = 15; col < 24; col++) { //Column header will look something like: abs_400 //Analyte ID will look like 400nm or 400nm post-acidification depending on acidified value in column L string inst_analyte_name = GetXLStringValue(worksheet.Cells[1, col]); string inst_analyte_num = inst_analyte_name.Split("_")[1].Trim(); string analyteID = ""; double measured_val = 0.0; if (acidified == 0) { analyteID = inst_analyte_num + "nm"; } else if (acidified == 1) { analyteID = inst_analyte_num + "nm " + analyte_post_acidification; } measured_val = GetXLDoubleValue(worksheet.Cells[row, col]); DataRow dr = dt.NewRow(); dr["Aliquot"] = aliquot; dr["Analyte Identifier"] = analyteID; dr["Measured Value"] = measured_val; dr["Dilution Factor"] = dilution_factor; dr["Analysis Date/Time"] = analysis_date; dr["User Defined 1"] = user_defined1; dt.Rows.Add(dr); } } rm.TemplateData = dt; } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } return(rm); }
public override DataTableResponseMessage Execute() { const string REMOVE = "’"; const string REMOVE2 = "â€"; string dateTime = ""; DataTableResponseMessage rm = new DataTableResponseMessage(); DataTable dt = null; try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); dt = GetDataTable(); FileInfo fi = new FileInfo(input_file); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; //This is a new way of using the 'using' keyword with braces using var package = new ExcelPackage(fi); var worksheet = package.Workbook.Worksheets[0]; //Worksheets are zero-based index string name = worksheet.Name; //File validation if (worksheet.Dimension == null) { string msg = string.Format("No data in Sheet 1 in InputFile: {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } int startRow = worksheet.Dimension.Start.Row; int startCol = worksheet.Dimension.Start.Column; int numRows = worksheet.Dimension.End.Row; int numCols = worksheet.Dimension.End.Column; //Date time is in row 1 columh H (8) if (worksheet.Cells[1, 8].Value == null) { throw new Exception($""); } dateTime = worksheet.Cells[1, 8].Value.ToString(); DataTable dtTemp = new DataTable(); //Get names of columns //Dont really need this but it constructs a datatable with correct number of columns for (int col = 1; col <= numCols; col++) { string colName = ""; if (worksheet.Cells[11, col].Value != null) { colName = worksheet.Cells[11, col].Value.ToString(); } string colName2 = ""; if (worksheet.Cells[12, col].Value != null) { colName2 = worksheet.Cells[12, col].Value.ToString(); } DataColumn dc = new DataColumn(colName + "-" + colName2); dtTemp.Columns.Add(dc); } //Loop through the data and push it into a datatable //Replace the first specified string with a single quote for (int row = 13; row <= numRows; row++) { DataRow dr = dtTemp.NewRow(); for (int col = 1; col <= numCols; col++) { string val = ""; if (worksheet.Cells[12, col].Value != null) { val = worksheet.Cells[row, col].Value.ToString(); val = val.Replace(REMOVE, "'"); //Remove all whitespaces in string val = val.Replace(" ", ""); } dr[col - 1] = val; } dtTemp.Rows.Add(dr); } //Loop through the datatable and replace the second specified string with two single quotes for (int row = 0; row < dtTemp.Rows.Count; row++) { for (int col = 0; col < dtTemp.Columns.Count; col++) { string val = dtTemp.Rows[row][col].ToString(); val = val.Replace(REMOVE2, "''"); //There is an issue with an extra space showing up in this string val = val.Replace(" ", ""); dtTemp.Rows[row][col] = val; } } for (int row = 0; row < dtTemp.Rows.Count; row++) { string analyteID = dtTemp.Rows[row][4].ToString(); string measuredVal = dtTemp.Rows[row][6].ToString().Trim(); string aliquot = dtTemp.Rows[row][1].ToString(); string userDefined1 = dtTemp.Rows[row][7].ToString(); string userDefined2 = dtTemp.Rows[row][10].ToString(); string userDefined3 = dtTemp.Rows[row][11].ToString(); string userDefined4 = dtTemp.Rows[row][12].ToString(); DataRow dr = dt.NewRow(); dr["Analyte Identifier"] = analyteID; dr["Measured Value"] = measuredVal; dr["Aliquot"] = aliquot; dr["Analysis Date/Time"] = dateTime; dr["User Defined 1"] = userDefined1; dr["User Defined 2"] = userDefined2; dr["User Defined 3"] = userDefined3; dr["User Defined 4"] = userDefined4; dt.Rows.Add(dr); } } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } rm.TemplateData = dt; return(rm); }
public override DataTableResponseMessage Execute() { DataTableResponseMessage rm = new DataTableResponseMessage(); DataTable dt = null; try { rm = VerifyInputFile(); if (rm != null) { return(rm); } rm = new DataTableResponseMessage(); dt = GetDataTable(); FileInfo fi = new FileInfo(input_file); dt.TableName = System.IO.Path.GetFileNameWithoutExtension(fi.FullName); //New in version 5 - must deal with License ExcelPackage.LicenseContext = LicenseContext.NonCommercial; //This is a new way of using the 'using' keyword with braces using var package = new ExcelPackage(fi); var worksheet = package.Workbook.Worksheets[0]; //Worksheets are zero-based index string name = worksheet.Name; //File validation if (worksheet.Dimension == null) { string msg = string.Format("No data in Sheet 1 in InputFile: {0}", input_file); rm.LogMessage = msg; rm.ErrorMessage = msg; return(rm); } int startRow = worksheet.Dimension.Start.Row; int startCol = worksheet.Dimension.Start.Column; int numRows = worksheet.Dimension.End.Row; int numCols = worksheet.Dimension.End.Column; //Build list of aliquots List <string> lstAliquots = new List <string>(); for (int row = 2; row <= numRows; row++) { string sval = GetXLStringValue(worksheet.Cells[row, 1]); if (string.IsNullOrWhiteSpace(sval)) { break; } lstAliquots.Add(sval); } int row_kingdom = 0; int row_phylum = 0; int row_class = 0; int row_order = 0; int row_family = 0; int row_genus = 0; int numAliquots = lstAliquots.Count; //Get description row range for (int row = numAliquots + 2; row < numRows + 1; row++) { string cell_val = Convert.ToString(worksheet.Cells[row, 1].Value); if (string.IsNullOrWhiteSpace(cell_val)) { continue; } cell_val = cell_val.Trim().ToLower(); if (cell_val == "kingdom") { row_kingdom = row; } else if (cell_val == "phylum") { row_phylum = row; } else if (cell_val == "class") { row_class = row; } else if (cell_val == "order") { row_order = row; } else if (cell_val == "family") { row_family = row; } else if (cell_val == "genus") { row_genus = row; } } // Build list of analyte ids List <string> lstAnalytes = new List <string>(); for (int col_idx = 2; col_idx <= numCols; col_idx++) { int row = 1; string analyte = GetXLStringValue(worksheet.Cells[row, col_idx]); if (string.IsNullOrWhiteSpace(analyte)) { break; } lstAnalytes.Add(analyte); } //Loop over data to get measured values List <string> data = new List <string>(); for (int row_idx = 2; row_idx <= lstAliquots.Count; row_idx++) { for (int col_idx = 2; col_idx <= lstAnalytes.Count; col_idx++) { DataRow row = dt.NewRow(); row["Aliquot"] = lstAliquots[row_idx - 2]; row["Analyte Identifier"] = lstAnalytes[col_idx - 2]; string measured_val = GetXLStringValue(worksheet.Cells[row_idx, col_idx]); //If meausured value is None set to 0 double dval; if (!string.IsNullOrWhiteSpace(measured_val)) { if (!double.TryParse(measured_val, out dval)) { measured_val = "0"; } } else { measured_val = "0"; } row["Measured Value"] = measured_val; string desc = GetXLStringValue(worksheet.Cells[row_kingdom, col_idx]) + ";"; desc += GetXLStringValue(worksheet.Cells[row_phylum, col_idx]) + ";"; desc += GetXLStringValue(worksheet.Cells[row_class, col_idx]) + ";"; desc += GetXLStringValue(worksheet.Cells[row_order, col_idx]) + ";"; desc += GetXLStringValue(worksheet.Cells[row_family, col_idx]) + ";"; desc += GetXLStringValue(worksheet.Cells[row_genus, col_idx]); row["Description"] = desc; dt.Rows.Add(row); } } } catch (Exception ex) { rm.LogMessage = string.Format("Processor: {0}, InputFile: {1}, Exception: {2}", name, input_file, ex.Message); rm.ErrorMessage = string.Format("Problem executing processor {0} on input file {1}.", name, input_file); } rm.TemplateData = dt; return(rm); }