public static Dictionary <int, Measurement> Convert(IODPDataTable dataTable, SectionInfoCollection SectionCollection)
{
_ = SectionCollection ?? throw new ArgumentNullException(nameof(SectionCollection));
_ = dataTable ?? throw new ArgumentNullException(nameof(dataTable));
Dictionary <int, Measurement> _measurements = new Dictionary <int, Measurement>();
int measurementCount = _measurements.Count + 1;
//TODO: Ignore record if error is thrown, ex: offsets with TCON
foreach (DataRow record in dataTable.DataTable.Rows)
{
SectionInfo measurementSectionInfo = new SectionInfo();
try
{
measurementSectionInfo.Expedition = record[dataTable.ExpeditionColumn].ToString();
measurementSectionInfo.Site = record[dataTable.SiteColumn].ToString();
measurementSectionInfo.Hole = record[dataTable.HoleColumn].ToString();
measurementSectionInfo.Core = record[dataTable.CoreColumn].ToString();
measurementSectionInfo.Type = record[dataTable.TypeColumn].ToString();
measurementSectionInfo.Section = record[dataTable.SectionColumn].ToString();
}
catch (Exception)
{
throw new IndexOutOfRangeException(nameof(record));
}
Measurement measurement = new Measurement(measurementSectionInfo);
measurement.SectionInfo = SectionCollection.GetExistingElseAddAndGetCurrentSection(measurement.SectionInfo);
//CARB files throw error here because there isn't an offset field within the file. Ensure there is.
try
{
if (!string.IsNullOrEmpty(dataTable.OffsetColumn))
{
measurement.StartOffset = double.Parse(record[dataTable.OffsetColumn].ToString(), CultureInfo.CurrentCulture);
measurement.EndOffset = double.Parse(record[dataTable.OffsetColumn].ToString(), CultureInfo.CurrentCulture);
}
if (!string.IsNullOrEmpty(dataTable.TopOffsetColumn))
{
measurement.StartOffset = double.Parse(record[dataTable.TopOffsetColumn].ToString(), CultureInfo.CurrentCulture);
}
if (!string.IsNullOrEmpty(dataTable.BottomOffsetColumn))
{
measurement.EndOffset = double.Parse(record[dataTable.BottomOffsetColumn].ToString(), CultureInfo.CurrentCulture);
}
measurement.DataRow = record;
_measurements.Add(measurementCount, measurement);
measurementCount++;
}
catch (Exception)
{
//throw;
}
}
return(_measurements);
}
public static void CheckFile(IODPDataTable IODPDataTable)
{
int rowNumber = 1;
foreach (DataRow row in IODPDataTable.DataTable.Rows)
{
if (!DataRowContainsDescription(row, IODPDataTable))
{
throw new Exception(string.Format("Row {0}: Does not contain description", rowNumber));
}
if (!DataRowContainsSampleIDColumn(row, IODPDataTable))
{
throw new Exception(string.Format("Row {0}: Does not contain SampleID column", rowNumber));
}
if (!DataRowContainsOffsetColumns(row, IODPDataTable))
{
throw new Exception(string.Format("Row {0}: Data row does not contain Offset columns", rowNumber));
}
if (!StartOffsetValuesAreValid(row, IODPDataTable))
{
throw new Exception(string.Format("Row {0}: Does not contain correct Start Offset values", rowNumber));
}
if (!EndOffsetValuesAreValid(row, IODPDataTable))
{
throw new Exception(string.Format("Row {0}: Does not contain correct End Offset values", rowNumber));
}
LithologicDescription Description = new LithologicDescription();
try
{
Description = new LithologicDescription(row[IODPDataTable.SampleIDColumn].ToString());
}
catch (Exception ex)
{
throw new Exception(string.Format("Row {0}: Could not parse SampleID. Error Message: {1}", rowNumber, ex.Message));
}
if (!DescriptionContainsSectionInfo(Description))
{
throw new Exception(string.Format("Row {0}: Description does not contain section information", rowNumber));
}
rowNumber++;
}
}
/// <summary>
/// Gets a collection of LithologicDescriptions from a corrected file.
/// </summary>
/// <param name="filename"></param>
/// <param name="columnIdentifiers"></param>
/// <returns></returns>
public static async Task <ICollection <LithologicDescription> > GetDescriptionsFromFileAsync(string filename, [Optional] IntervalHierarchyNames columnIdentifiers)
{
columnIdentifiers = columnIdentifiers ?? new IntervalHierarchyNames()
{
Expedition = "Expedition_VP",
Site = "Site_VP",
Hole = "Hole_VP",
Core = "Core_VP",
Type = "Type_VP",
Section = "Section_VP",
Half = "SectionHalf_VP",
TopOffset = "TopOffset_VP",
BottomOffset = "BottomOffset_VP",
ArchiveTextID = "ArchiveSectionTextID_VP",
WorkingTextID = "WorkingSectionTextID_VP",
ParentTextID = "ParentSectionTextID_VP"
};
IODPDataTable iODPDataTable = Importer.ImportDataTableFromFile(filename, columnIdentifiers);
ICollection <LithologicDescription> descriptions = new HashSet <LithologicDescription>();
using (DescDBContext dbContext = new DescDBContext())
{
try
{
foreach (DataRow row in iODPDataTable.DataTable.Rows)
{
SectionInfo section = new SectionInfo(Importer.GetHierarchyValuesFromDataRow(row, columnIdentifiers));
LithologicDescription description = new LithologicDescription();
// description.SectionInfo = section;
description.SectionInfo = await DatabaseWorkflowHandler.GetSectionInfoFromDatabaseForIntervalAsync(dbContext, section).ConfigureAwait(true);
description.LithologicID = row["LithologicID_VP"].ToString();
description.DataRow = row;
description.DescriptionReport = row["Filename_VP"].ToString();;
description.StartOffset = double.TryParse(row[columnIdentifiers.TopOffset].ToString(), out double startOffset) ? startOffset : -1;
description.EndOffset = double.TryParse(row[columnIdentifiers.BottomOffset].ToString(), out double endOffset) ? endOffset : -1;
descriptions.Add(description);
}
}
catch (Exception)
{
throw new Exception("Error creating lithologic description from data row");
}
}
return(descriptions);
}
public static void AddSampleIDColumnToFileAndExport(string path, string newPath)
{
//Impor a file:
CSVReader reader = new CSVReader();
reader.ReadPath = path;
using (DataTable importTable = reader.Read())
{
IntervalHierarchyNames columnNames = new IntervalHierarchyNames()
{
Expedition = "Exp",
Site = "Site",
Hole = "Hole",
Core = "Core",
Type = "Type",
Section = "Section",
Half = "A/W",
};
IODPDataTable iODPDataTable = new IODPDataTable(importTable, columnNames);
if (iODPDataTable.DataTable.Columns.Contains("Sample") || iODPDataTable.DataTable.Columns.Contains("Label ID"))
{
return;
}
if (!iODPDataTable.DataTable.Columns.Contains("Sample"))
{
iODPDataTable.DataTable.Columns.Add("Sample").SetOrdinal(1);
}
foreach (DataRow row in importTable.Rows)
{
row.BeginEdit();
row["Sample"] = $"{row[iODPDataTable.ExpeditionColumn]}" +
$"-{row[iODPDataTable.SiteColumn]}{row[iODPDataTable.HoleColumn]}" +
$"-{row[iODPDataTable.CoreColumn]}{row[iODPDataTable.TypeColumn]}" +
$"-{row[iODPDataTable.SectionColumn]}" +
$"-{row[columnNames.Half]}";
row.EndEdit();
}
//Export File
Importer.ExportDataTableAsNewFile(newPath, importTable);
}
}
private static void AddDataToHierarchyColumns(IODPDataTable IODPDataTable, string fileName, DataTable allSectionsDataTable)
{
int rowNumber = 1;
IntervalHierarchyNames sectionTableColumnNames = new IntervalHierarchyNames()
{
Expedition = "Exp",
Site = "Site",
Hole = "Hole",
Core = "Core",
Type = "Type",
Section = "Sect"
};
foreach (DataRow row in IODPDataTable.DataTable.Rows)
{
LithologicDescription description = new LithologicDescription(row[IODPDataTable.SampleIDColumn].ToString());
double parsedOffset = 0;
Importer.StartOffsetValuesAreValid(row, IODPDataTable, ref parsedOffset);
description.StartOffset = parsedOffset;
Importer.EndOffsetValuesAreValid(row, IODPDataTable, ref parsedOffset);
description.EndOffset = parsedOffset;
LithologicIDGenerator idGenerator = new LithologicIDGenerator();
var textids = GetSectionTextIDsForDescription(allSectionsDataTable, description, sectionTableColumnNames);
try
{
var descriptionID = idGenerator.GenerateID(description);
row.BeginEdit();
row["Filename_VP"] = fileName;
row["LithologicID_VP"] = descriptionID;
row["ArchiveSectionTextID_VP"] = textids.Archive;
row["WorkingSectionTextID_VP"] = textids.Working;
row["ParentSectionTextID_VP"] = textids.Parent;
row["Expedition_VP"] = description.SectionInfo.Expedition;
row["Site_VP"] = description.SectionInfo.Site;
row["Hole_VP"] = description.SectionInfo.Hole;
row["Core_VP"] = description.SectionInfo.Core;
row["Type_VP"] = description.SectionInfo.Type;
row["Section_VP"] = description.SectionInfo.Section;
row["SectionHalf_VP"] = description.SectionInfo.Half;
row["TopOffset_VP"] = description.StartOffset.ToString();
row["BottomOffset_VP"] = description.EndOffset.ToString();
row.EndEdit();
}
catch (Exception)
{
Log.Warning(string.Format("Row {0}: Unable to populate data for description", rowNumber.ToString()));
}
rowNumber++;
}
}
public static bool DataRowContainsSampleIDColumn(DataRow dataTableRow, IODPDataTable dataTable)
{
return(dataTableRow.Table.Columns.Contains(dataTable.SampleIDColumn));
}
public static bool DataRowContainsDescription(DataRow dataTableRow, IODPDataTable dataTable)
{
CinnamonList noDataEntriesList = new CinnamonList("NoSampleEntries");
return(!noDataEntriesList.FindInList(dataTableRow[dataTable.SampleIDColumn].ToString().ToLower()) ? true : false);
}
public static bool DataRowContainsOffsetColumns(DataRow dataTableRow, IODPDataTable dataTable)
{
return(dataTableRow.Table.Columns.Contains(dataTable.TopOffsetColumn) && dataTableRow.Table.Columns.Contains(dataTable.BottomOffsetColumn));
}
public static bool EndOffsetValuesAreValid(DataRow dataTableRow, IODPDataTable dataTable)
{
return(double.TryParse(dataTableRow[dataTable.BottomOffsetColumn].ToString(), out double parsedOffset));
}
public static bool StartOffsetValuesAreValid(DataRow dataTableRow, IODPDataTable dataTable, ref double parsedOffset)
{
return(double.TryParse(dataTableRow[dataTable.TopOffsetColumn].ToString(), out parsedOffset));
}
/// <summary>
/// Asynchronously gets a collection of IODP measurements from a .csv file.
/// </summary>
/// <param name="filename">The .csv file location</param>
/// <param name="columnIdentifiers">Optional parameter which specifies the file's column names</param>
/// <returns>A collection of measurements</returns>
public static async Task <ICollection <Measurement> > GetMeasurementsFromFileAsync(string filename, [Optional] IntervalHierarchyNames columnIdentifiers)
{
columnIdentifiers = columnIdentifiers ?? new IntervalHierarchyNames()
{
Expedition = "Exp",
Site = "Site",
Hole = "Hole",
Core = "Core",
Type = "Type",
Section = "Sect",
Half = "A/W",
TopOffset = "Offset (cm)",
BottomOffset = "Offset (cm)",
ArchiveTextID = "ArchiveSectionTextID_VP",
WorkingTextID = "WorkingSectionTextID_VP",
ParentTextID = "ParentSectionTextID_VP",
SampleID = "Sample",
TextID = "Text ID",
TestNumber = "Test No.",
};
//TODO: need to get this some other way
string InstrumentSystem = Importer.GetFileNameWithoutExtension(filename);
if (InstrumentSystem == "CARB" || InstrumentSystem == "ICP")
{
columnIdentifiers.TopOffset = "Top offset on section (cm)";
columnIdentifiers.BottomOffset = "Bot offset on section (cm)";
}
IODPDataTable iODPDataTable = Importer.ImportDataTableFromFile(filename, columnIdentifiers);
ICollection <Measurement> measurements = new HashSet <Measurement>();
try
{
foreach (DataRow row in iODPDataTable.DataTable.Rows)
{
IntervalHierarchyValues parsedValues = Importer.GetHierarchyValuesFromDataRow(row, columnIdentifiers);
Measurement measurement = new Measurement();
measurement.SectionInfo = new SectionInfo(parsedValues); //Creating a SectionInfo here that will be used to find the one stored in the DB.
measurement.DataRow = row;
measurement.InstrumentReport = "";
measurement.InstrumentSystem = InstrumentSystem;
measurement.TextID = parsedValues.TextID;
measurement.TestNumber = parsedValues.TestNumber;
measurement.StartOffset = double.TryParse(row[columnIdentifiers.TopOffset].ToString(), out double startOffset) ? startOffset : -1;
measurement.EndOffset = double.TryParse(row[columnIdentifiers.BottomOffset].ToString(), out double endOffset) ? endOffset : -1;
measurements.Add(measurement);
}
}
catch (Exception)
{
throw new Exception("Error creating measurement from data row");
}
using (DescDBContext dbContext = new DescDBContext())
{
string[] expeditions = measurements.Select(x => x.SectionInfo.Expedition).Distinct().ToArray();
ICollection <SectionInfo> sections;
try
{
sections = await DatabaseWorkflowHandler.GetAllSectionsFromDatabaseForExpeditionAsync(dbContext, expeditions).ConfigureAwait(false);
}
catch (Exception)
{
throw new Exception("Could not get sections from the database");
}
foreach (var measurement in measurements)
{
measurement.SectionInfo = DatabaseWorkflowHandler.GetSectionInfoFromCollection(sections, measurement.SectionInfo);
}
return(measurements);
}
}
/// <summary>
/// Converts an IODPDataTable object into a collection of Lithologic Descriptions
/// </summary>
/// <param name="dataTable">The datatable to convert</param>
/// <returns></returns>
public static Dictionary <string, LithologicDescription> ConvertDatatableToDictionary(IODPDataTable dataTable, SectionInfoCollection SectionCollection)
{
_ = SectionCollection ?? throw new ArgumentNullException(nameof(SectionCollection));
var LithologyCache = new Dictionary <string, LithologicDescription>();
if (dataTable == null)
{
return(LithologyCache);
}
//Add a column in the datatable to ensure consistency between files with and without descriptions:
dataTable.DataTable.Columns.Add("LithologicID_VP", typeof(string)).SetOrdinal(0);
foreach (DataRow dataTableRow in dataTable.DataTable.Rows)
{
dataTableRow["LithologicID_VP"] = "-1";
if (!Importer.DataRowContainsDescription(dataTableRow, dataTable))
{
return(LithologyCache);
}
if (!Importer.DataRowContainsSampleIDColumn(dataTableRow, dataTable))
{
return(LithologyCache);
}
LithologicDescription description = new LithologicDescription(dataTableRow[dataTable.SampleIDColumn].ToString());
description.SectionInfo = SectionCollection.GetExistingElseAddAndGetCurrentSection(description.SectionInfo);
if (!Importer.DescriptionContainsSectionInfo(description))
{
return(LithologyCache);
}
description.DataRow = dataTableRow;
double parsedOffset = 0;
if (!Importer.DataRowContainsOffsetColumns(dataTableRow, dataTable))
{
return(LithologyCache);
}
if (!Importer.StartOffsetValuesAreValid(dataTableRow, dataTable, ref parsedOffset))
{
return(LithologyCache);
}
description.StartOffset = parsedOffset;
if (!Importer.EndOffsetValuesAreValid(dataTableRow, dataTable, ref parsedOffset))
{
return(LithologyCache);
}
description.EndOffset = parsedOffset;
LithologicIDGenerator IDGenerator = new LithologicIDGenerator();
IDGenerator.GenerateID(description);
if (description.OffsetsSet())
{
description.GenerateSubintervals();
}
description.DataRow["LithologicID_VP"] = description.LithologicID;
//Some descriptions are split in two rows. It's very uncommon, but throws an error
//Only selecting the first row, despite the loss of data
if (!LithologyCache.ContainsKey(description.LithologicID))
{
LithologyCache.Add(description.LithologicID, description);
}
}
return(LithologyCache);
}
