/// <summary>
/// Load a file path into a DataVariable. Does NOT add it to the DataManager
/// </summary>
/// <param name="hasRowHeaders">Flag. Set if data file is expected to have row headers.</param>
/// <param name="hasColumnHeaders">Flag. Set if data file is expected to have column headers.</param>
/// <param name="dataVariable">Returns a new DataVariable. Is valid but empty object if error or canceled.</param>
/// <param name="errorMsg">Contains an error message on failure.</param>
/// <returns>True on success. False if user cancels file picker or if there's an error.</returns>
public bool LoadFile(string path, bool hasRowHeaders, bool hasColumnHeaders, out DataVariable dataVariable, out string errorMsg)
{
dataVariable = new DataVariable();
//foreach (string s in path)
// Debug.Log(s);
bool success = false;
errorMsg = "No Error";
//Cast to base class for reading in the file
CSVReaderData data = (CSVReaderData)dataVariable; // new CSVReaderData();
try
{
success = CSVReader.Read(path, hasColumnHeaders, hasRowHeaders, ref data, out errorMsg);
}
catch (Exception e)
{
errorMsg = "Exception caught: " + e.ToString();
Debug.Log(errorMsg);
return(false);
}
if (success)
{
dataVariable.Filepath = path;
}
else
{
Debug.Log("Error msg from csv read: \n");
Debug.Log(errorMsg);
}
return(success);
}
// Update is called once per frame
void Update()
{
if (Input.GetKeyDown(KeyCode.O))
{
string[] paths = StandaloneFileBrowser.OpenFilePanel("Open File", "", "", true);
foreach (string s in paths)
{
Debug.Log(s);
}
if (paths.Length == 0)
{
return;
}
bool success = false;
string errorMsg = "Unknown Error";
//CSVReaderData data = new CSVReaderData();
DataVariable dataVariable = new DataVariable();
CSVReaderData dataObj = (CSVReaderData)dataVariable; // new CSVReaderData();
try
{
success = CSVReader.Read(paths[0], hasColumnsHeader, hasRowHeader, ref dataObj, out errorMsg);
}
catch (Exception e)
{
Debug.Log("Exception caugt: " + e.ToString());
return;
}
if (!success)
{
Debug.Log("Error msg from csv read: ");
Debug.Log(errorMsg);
}
else
{
dataVariable.DumpMetaData();
dataVariable.DumpData();
dataVariable.Clear();
}
}
}
/// <summary>
/// Read the csv or tab-delimited file specified by 'file'.
/// </summary>
/// <param name="file">Full path for file to read.</param>
/// <param name="columnHeadersExpected">Flag. Set this if columns headers are expected in the file. This is headers in the first row/line of the file.</param>
/// <param name="rowHeadersExpected">Flag. Set this is row headers are expected. This means the first field/column of each row is a header/name/string for the row</param>
/// <param name="result">Ref var. If passed object is non-null, it's filled, otherwise a new data object - containing data, headers, etc.</param>
/// <param name="errorMsg">Return value. Error message string if failed (including exception string if an excpetion occured).</param>
/// <returns>True on success, otherwise false</returns>
public static bool Read(string file, bool columnHeadersExpected, bool rowHeadersExpected, ref CSVReaderData result, out string errorMsg)
{
errorMsg = "no error";
//For matching no data - NaN and None. This list (initially at least) from what python recognizes according to https://stackoverflow.com/questions/46612576/counting-number-of-nan-not-zeros-or-blanks-in-csv
//NOTE - we'll do a case insensitive comparison
string[] noDataStrings = new string[] { "none", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-nan", "1.#IND", "1.#QNAN", "N/A", "NA", "NULL", "nan" };
if (result == null)
{
result = new CSVReaderData();
}
//var list = new List<Dictionary<string, object>>();
//TextAsset textData = Resources.Load(file) as TextAsset;
//string[] lines = Regex.Split(textData.text, LINE_SPLIT_RE);
//NOTE - reads whole file into memory. If we get into big data, we'll want
// to read line-by-line. See https://stackoverflow.com/questions/46405067/reading-a-file-line-by-line-in-c-sharp-using-streamreader
//NOTE - will this work with non-windows line endings?
string[] lines;
try
{
//NOTE - reads in all lines at once. Won't be good for very large files.
lines = File.ReadAllLines(file);
}
catch (Exception e)
{
errorMsg = "Failed loading file: " + file + ". Exception: " + e.ToString();
Debug.Log(errorMsg);
result.Clear();
return(false);
}
if (lines.Length < 1)
{
result.Clear();
return(false);
}
int numDataColumns = 0;
int numDataRows = 0;
result.hasRowHeaders = rowHeadersExpected;
result.hasColumnHeaders = columnHeadersExpected;
//number of data rows
result.numDataRows = numDataRows = lines.Length - (columnHeadersExpected ? 1 : 0);
//number of data columns and columns headers
var firstRow = Regex.Split(lines[0], SPLIT_RE);
int numAllColumns = firstRow.Length;
if (columnHeadersExpected)
{
//If we have row headers, skip the first value since it's not a data-column header
int ind = rowHeadersExpected ? 1 : 0;
for (int i = ind; i < firstRow.Length; i++)
{
result.columnHeaders.Add(firstRow[i]);
}
}
result.numDataCols = numDataColumns = firstRow.Length - (rowHeadersExpected ? 1 : 0);
//Alloc the data array
try
{
result.Data = new float[numDataRows][];
for (int row = 0; row < numDataRows; row++)
{
result.Data[row] = new float[numDataColumns];
}
}
catch (Exception e)
{
errorMsg = "Aborting. Failed allocating data array, with exception: " + e.ToString();
Debug.Log(errorMsg);
result.Clear();
return(false);
}
//Parse the data lines
int start = columnHeadersExpected ? 1 : 0;
for (var fileRow = start; fileRow < lines.Length; fileRow++)
{
var values = Regex.Split(lines[fileRow], SPLIT_RE);
//or try string.split for simpler: https://docs.microsoft.com/en-us/dotnet/csharp/how-to/parse-strings-using-split
//Empty line or not enough columns? Abort.
if (values.Length != numAllColumns)
{
errorMsg = "Row " + fileRow + " is empty or otherwise incorrect length: " + values.Length + " instead of " + numAllColumns + ". Aborting.";
Debug.Log(errorMsg);
result.Clear();
return(false);
}
//Parse a line
for (var fileCol = 0; fileCol < numAllColumns; fileCol++)
{
string value = values[fileCol];
int ii = fileRow - (columnHeadersExpected ? 1 : 0);
int jj = fileCol - (rowHeadersExpected ? 1 : 0);
//empty data cell? Call it NaN if we're not expecting row or columns headers, or we're past the first column
if (value == "" && ((rowHeadersExpected && fileCol > 0) || !rowHeadersExpected || !columnHeadersExpected))
{
/* were not supporting NaN originally
* errorMsg = "Empty data cell. Row, col: " + fileRow + ", " + fileCol + ". Aborting.";
* Debug.Log(errorMsg);
* result.Clear();
* return false;
*/
result.Data[ii][jj] = float.NaN;
continue;
}
//Trim chars in TRIM_CHARS, remove trailing and leading white space, replace \
value = value.Trim(TRIM_CHARS).Trim().Replace("\\", "");
if (rowHeadersExpected && fileCol == 0)
{
//Header
result.rowHeaders.Add(value);
continue;
}
//data
float f;
if (float.TryParse(value, out f))
{
result.Data[ii][jj] = f;
}
else
{
//first check for NaN or None (note that "NaN" may be parsed by TryParse above)
bool foundNaN = false;
foreach (string s in noDataStrings)
{
if (value.Equals(s, StringComparison.CurrentCultureIgnoreCase))
{
result.Data[ii][jj] = float.NaN;
foundNaN = true;
break;
}
}
if (foundNaN)
{
continue;
}
//error
string ex = "";
if (fileRow == 0 && !columnHeadersExpected)
{
ex = "Wasn't expecting first row to be headers. But is it maybe? ";
}
else if (fileCol == 0 && !rowHeadersExpected)
{
ex = "Wasn't expecting first columns to be headers. But is it maybe? ";
}
errorMsg = ex + "Expected a number but got non-numeric value '" + value + "', at row, col: " + fileRow + ", " + fileCol + ". Aborting.";
Debug.Log(errorMsg);
result.Clear();
return(false);
}
}
}
return(true);
}
