/// <summary>
/// Check if the data point is of extreme values. For example if the data point is furthest away in a direction.
/// </summary>
/// <param name="filename">Data point</param>
void CheckIfEdgeData(LabeledData data)
{
if (firstData)
{
farAwayNegX = data.Position.x;
farAwayPosX = data.Position.x;
farAwayNegY = data.Position.y;
farAwayPosY = data.Position.y;
farAwayNegZ = data.Position.z;
farAwayPosZ = data.Position.z;
firstData = false;
}
//Multidata
for (int i = 0; i < data.features.Length; i++)
{
if (data.features[i] > maxData[i])
{
maxData[i] = data.features[i];
}
if (data.features[i] < minData[i])
{
minData[i] = data.features[i];
}
accumulatedData[i] += data.features[i];
}
if (data.Position.x < farAwayNegX)
{
farAwayNegX = data.Position.x;
}
if (data.Position.x > farAwayPosX)
{
farAwayPosX = data.Position.x;
}
if (data.Position.y < farAwayNegY)
{
farAwayNegY = data.Position.y;
}
if (data.Position.y > farAwayPosY)
{
farAwayPosY = data.Position.y;
}
if (data.Position.z < farAwayNegZ)
{
farAwayNegZ = data.Position.z;
}
if (data.Position.z > farAwayPosZ)
{
farAwayPosZ = data.Position.z;
}
}
public void LoadData()
{
LabeledData <float> shapeData = LoadData(FitToShape);
LabeledData <float> toAggregate = LoadData(DataToAggregate);
LabeledData <float> ret = new LabeledData <float>();
// add all of the keys with a zero value
foreach (var e in shapeData)
{
ret.Add(e.Key, 0.0f);
}
if (DataMap != null)
{
//Load in the map
using (var reader = new CsvReader(DataMap, true))
{
//burn header
reader.LoadLine();
while (reader.LoadLine(out int columns))
{
if (columns >= 3)
{
reader.Get(out string destName, 0);
reader.Get(out string originName, 1);
reader.Get(out float toApply, 2);
if (!ret.TryGetValue(destName, out float destValue))
{
continue;
}
if (!toAggregate.TryGetValue(originName, out float originValue))
{
continue;
}
ret[destName] = originValue * toApply + destValue;
}
}
}
}
else
{
var keys = ret.Keys.ToList();
foreach (var key in keys)
{
if (toAggregate.TryGetValue(key, out float data))
{
ret[key] = data;
}
}
}
_Data = ret;
Loaded = true;
}
private void Add <TData>(LabeledData <TData> set, string label, TData data)
{
switch (AggregationToApply)
{
case Agg.None:
if (set.ContainsKey(label))
{
throw new XTMFRuntimeException(this, $"In '{Name}' while loading in labeled data a label was loaded multiple times '{label}'!");
}
set.Add(label, data);
break;
case Agg.Sum:
if (typeof(TData) == typeof(float))
{
// the optimizer should be able to solve this
var fData = (float)(object)data;
var fSet = set as LabeledData <float>;
// ReSharper disable once PossibleNullReferenceException
fSet.TryGetValue(label, out float alreadyContained);
fSet[label] = fData + alreadyContained;
}
break;
case Agg.Multiply:
if (typeof(TData) == typeof(float))
{
// the optimizer should be able to solve this
var fData = (float)(object)data;
var fSet = set as LabeledData <float>;
// ReSharper disable once PossibleNullReferenceException
if (!fSet.TryGetValue(label, out float alreadyContained))
{
alreadyContained = 1.0f;
}
fSet[label] = fData * alreadyContained;
}
break;
case Agg.Count:
if (typeof(TData) == typeof(float))
{
var fSet = set as LabeledData <float>;
// ReSharper disable once PossibleNullReferenceException
fSet.TryGetValue(label, out float alreadyContained);
fSet[label] = 1 + alreadyContained;
}
break;
}
}
public void LoadData()
{
var ret = new LabeledData <T>();
using (var reader = new CsvReader(LoadFrom, true))
{
//burn the header
reader.LoadLine();
string error = null;
int lineNumber = 0;
// load the data
while (reader.LoadLine(out int columns))
{
lineNumber++;
if (columns >= 2)
{
reader.Get(out string label, 0);
if (typeof(T) == typeof(float))
{
LabeledData <float> fRet = ret as LabeledData <float>;
reader.Get(out float parsedData, 1);
Add(fRet, label, parsedData);
}
else
{
reader.Get(out string data, 1);
var parsedData = ArbitraryParameterParser.ArbitraryParameterParse(typeof(T), data, ref error);
if (parsedData == null || error != null)
{
throw new XTMFRuntimeException(this, $"In '{Name}' we were unable to parse the data in line number {lineNumber}!\r\n{error}");
}
Add(ret, label, (T)parsedData);
}
}
}
}
_Data = ret;
Loaded = true;
}
static int SortByValue(LabeledData d1, LabeledData d2)
{
return(d1.features[0].CompareTo(d2.features[0]));
}
private void LoadInData(string dPath)
{
// Load data from text file (no headers, line 1 is data with whitespaces)
Debug.Log(Application.dataPath + "/" + "starparticles.001.off");
#if UNITY_EDITOR
StreamReader sr = new StreamReader(Application.dataPath + "/" + dPath);
#else
Debug.Log(Application.dataPath + "/" + "starparticles.001.off");
StreamReader sr = new StreamReader(dPath);
#endif
string[] buffer;
string line;
line = sr.ReadLine();
buffer = line.Split(null);
for (int fish = 0; fish < buffer.Length; fish++)
{
Debug.Log(buffer[fish]);
}
//HG: An unaccurate line counter (is bigger):
numLines = (int)((sr.BaseStream.Length) / sr.ReadLine().Length);
//HG: An unaccurate but slower counter:
if (countLines)
{
numLines = 0;
while (!sr.EndOfStream)
{
line = sr.ReadLine();
numLines += 1;
}
numPoints = numLines;
#if UNITY_EDITOR
sr = new StreamReader(Application.dataPath + "/" + dPath);
#else
sr = new StreamReader(dPath);
#endif
}
points = new Vector3[numLines];
minValue = new Vector3();
int i = 0;
dataPoints.Capacity = numLines;
featureLength = (labelNames.Length);
ResetValues();
unitNames = new string[buffer.Length];
//Debug.Log(buffer.Length);
for (i = 0; i < labelNames.Length; i++)
{
if (i == 0)
{
MenuHandler.Instance.graphVariableText.GetComponent <Text>().text = labelNames[i];
}
}
sr.ReadLine();
sr.ReadLine();
String fish2 = sr.ReadLine().Trim();
buffer = fish2.Split(null);
Debug.Log("featLength: " + featureLength);
i = 0;
string flag = "";
int count = 0;
while (!sr.EndOfStream && i < numLines)
{
try
{
flag = "point";
if (!invertYZ)
{
points[i] = new Vector3(float.Parse(buffer[1]) * scale - offSet, float.Parse(buffer[2]) * scale - offSet, float.Parse(buffer[3]) * scale - offSet);
}
else
{
points[i] = new Vector3(float.Parse(buffer[0]) * scale - offSet, float.Parse(buffer[1]) * scale - offSet, float.Parse(buffer[2]) * scale - offSet);
}
if (buffer.Length >= 3) //HG: Chrashing after !
{
LabeledData d = new LabeledData();
d.Label = i.ToString();
d.Color = new Color(0, 0, 0);
d.Position = points[i];
//Multidata
int j = 0;
d.features = new float[featureLength];
while (j < featureLength)
{
try {
d.features[j] = float.Parse(buffer[j]);
} catch (Exception e)
{
Debug.Log("this failed: " + buffer[j]);
//We don't seem to like negative values in our document
}
accumulatedData[j] += d.features[j];
j++;
}
if (count == 0)
{
for (j = 0; j < labelNames.Length; j++)
{
Debug.Log(j + ": " + d.features[j]);
}
}
dataPoints.Add(d);
CheckIfEdgeData(d);
count++;
}
}
catch (IOException)
{
Debug.Log("Reading data interrupted at line " + i.ToString() + " at " + flag);
}
numPoints = numLines;
// GUI progress:
progress = i * 1.0f / (numLines - 1) * 1.0f;
if (i % Mathf.FloorToInt(numLines / 20) == 0)
{
guiText = i.ToString() + " out of " + numLines.ToString() + " loaded!!";
//TODO removed yield will it break anything?
}
fish2 = sr.ReadLine().Trim();
buffer = fish2.Split(null);
i += 1;
}
CalculateStdAndPercentiles(dataPoints);
numLines = i;
maxSizeTotal = new List <float>(topPercentile).ToArray();
currentDataPoints = new List <LabeledData>(dataPoints);
}
/// <summary>
/// Override only when overriding the normalization functions isn't enough.
/// </summary>
protected virtual LabeledData DataAsLabeledData(T[] data, L[] label)
{
double[] input = NormalizeInputData(data);
double[] output = NormalizeOutputData(label);
return(LabeledData.MakeFromArrays(input, output));
}
public void UnloadData()
{
Loaded = false;
_Data = null;
}
