/// <summary> /// Check if the data point is of extreme values. For example if the data point is furthest away in a direction. /// </summary> /// <param name="filename">Data point</param> void CheckIfEdgeData(LabeledData data) { if (firstData) { farAwayNegX = data.Position.x; farAwayPosX = data.Position.x; farAwayNegY = data.Position.y; farAwayPosY = data.Position.y; farAwayNegZ = data.Position.z; farAwayPosZ = data.Position.z; firstData = false; } //Multidata for (int i = 0; i < data.features.Length; i++) { if (data.features[i] > maxData[i]) { maxData[i] = data.features[i]; } if (data.features[i] < minData[i]) { minData[i] = data.features[i]; } accumulatedData[i] += data.features[i]; } if (data.Position.x < farAwayNegX) { farAwayNegX = data.Position.x; } if (data.Position.x > farAwayPosX) { farAwayPosX = data.Position.x; } if (data.Position.y < farAwayNegY) { farAwayNegY = data.Position.y; } if (data.Position.y > farAwayPosY) { farAwayPosY = data.Position.y; } if (data.Position.z < farAwayNegZ) { farAwayNegZ = data.Position.z; } if (data.Position.z > farAwayPosZ) { farAwayPosZ = data.Position.z; } }
public void LoadData() { LabeledData <float> shapeData = LoadData(FitToShape); LabeledData <float> toAggregate = LoadData(DataToAggregate); LabeledData <float> ret = new LabeledData <float>(); // add all of the keys with a zero value foreach (var e in shapeData) { ret.Add(e.Key, 0.0f); } if (DataMap != null) { //Load in the map using (var reader = new CsvReader(DataMap, true)) { //burn header reader.LoadLine(); while (reader.LoadLine(out int columns)) { if (columns >= 3) { reader.Get(out string destName, 0); reader.Get(out string originName, 1); reader.Get(out float toApply, 2); if (!ret.TryGetValue(destName, out float destValue)) { continue; } if (!toAggregate.TryGetValue(originName, out float originValue)) { continue; } ret[destName] = originValue * toApply + destValue; } } } } else { var keys = ret.Keys.ToList(); foreach (var key in keys) { if (toAggregate.TryGetValue(key, out float data)) { ret[key] = data; } } } _Data = ret; Loaded = true; }
private void Add <TData>(LabeledData <TData> set, string label, TData data) { switch (AggregationToApply) { case Agg.None: if (set.ContainsKey(label)) { throw new XTMFRuntimeException(this, $"In '{Name}' while loading in labeled data a label was loaded multiple times '{label}'!"); } set.Add(label, data); break; case Agg.Sum: if (typeof(TData) == typeof(float)) { // the optimizer should be able to solve this var fData = (float)(object)data; var fSet = set as LabeledData <float>; // ReSharper disable once PossibleNullReferenceException fSet.TryGetValue(label, out float alreadyContained); fSet[label] = fData + alreadyContained; } break; case Agg.Multiply: if (typeof(TData) == typeof(float)) { // the optimizer should be able to solve this var fData = (float)(object)data; var fSet = set as LabeledData <float>; // ReSharper disable once PossibleNullReferenceException if (!fSet.TryGetValue(label, out float alreadyContained)) { alreadyContained = 1.0f; } fSet[label] = fData * alreadyContained; } break; case Agg.Count: if (typeof(TData) == typeof(float)) { var fSet = set as LabeledData <float>; // ReSharper disable once PossibleNullReferenceException fSet.TryGetValue(label, out float alreadyContained); fSet[label] = 1 + alreadyContained; } break; } }
public void LoadData() { var ret = new LabeledData <T>(); using (var reader = new CsvReader(LoadFrom, true)) { //burn the header reader.LoadLine(); string error = null; int lineNumber = 0; // load the data while (reader.LoadLine(out int columns)) { lineNumber++; if (columns >= 2) { reader.Get(out string label, 0); if (typeof(T) == typeof(float)) { LabeledData <float> fRet = ret as LabeledData <float>; reader.Get(out float parsedData, 1); Add(fRet, label, parsedData); } else { reader.Get(out string data, 1); var parsedData = ArbitraryParameterParser.ArbitraryParameterParse(typeof(T), data, ref error); if (parsedData == null || error != null) { throw new XTMFRuntimeException(this, $"In '{Name}' we were unable to parse the data in line number {lineNumber}!\r\n{error}"); } Add(ret, label, (T)parsedData); } } } } _Data = ret; Loaded = true; }
static int SortByValue(LabeledData d1, LabeledData d2) { return(d1.features[0].CompareTo(d2.features[0])); }
private void LoadInData(string dPath) { // Load data from text file (no headers, line 1 is data with whitespaces) Debug.Log(Application.dataPath + "/" + "starparticles.001.off"); #if UNITY_EDITOR StreamReader sr = new StreamReader(Application.dataPath + "/" + dPath); #else Debug.Log(Application.dataPath + "/" + "starparticles.001.off"); StreamReader sr = new StreamReader(dPath); #endif string[] buffer; string line; line = sr.ReadLine(); buffer = line.Split(null); for (int fish = 0; fish < buffer.Length; fish++) { Debug.Log(buffer[fish]); } //HG: An unaccurate line counter (is bigger): numLines = (int)((sr.BaseStream.Length) / sr.ReadLine().Length); //HG: An unaccurate but slower counter: if (countLines) { numLines = 0; while (!sr.EndOfStream) { line = sr.ReadLine(); numLines += 1; } numPoints = numLines; #if UNITY_EDITOR sr = new StreamReader(Application.dataPath + "/" + dPath); #else sr = new StreamReader(dPath); #endif } points = new Vector3[numLines]; minValue = new Vector3(); int i = 0; dataPoints.Capacity = numLines; featureLength = (labelNames.Length); ResetValues(); unitNames = new string[buffer.Length]; //Debug.Log(buffer.Length); for (i = 0; i < labelNames.Length; i++) { if (i == 0) { MenuHandler.Instance.graphVariableText.GetComponent <Text>().text = labelNames[i]; } } sr.ReadLine(); sr.ReadLine(); String fish2 = sr.ReadLine().Trim(); buffer = fish2.Split(null); Debug.Log("featLength: " + featureLength); i = 0; string flag = ""; int count = 0; while (!sr.EndOfStream && i < numLines) { try { flag = "point"; if (!invertYZ) { points[i] = new Vector3(float.Parse(buffer[1]) * scale - offSet, float.Parse(buffer[2]) * scale - offSet, float.Parse(buffer[3]) * scale - offSet); } else { points[i] = new Vector3(float.Parse(buffer[0]) * scale - offSet, float.Parse(buffer[1]) * scale - offSet, float.Parse(buffer[2]) * scale - offSet); } if (buffer.Length >= 3) //HG: Chrashing after ! { LabeledData d = new LabeledData(); d.Label = i.ToString(); d.Color = new Color(0, 0, 0); d.Position = points[i]; //Multidata int j = 0; d.features = new float[featureLength]; while (j < featureLength) { try { d.features[j] = float.Parse(buffer[j]); } catch (Exception e) { Debug.Log("this failed: " + buffer[j]); //We don't seem to like negative values in our document } accumulatedData[j] += d.features[j]; j++; } if (count == 0) { for (j = 0; j < labelNames.Length; j++) { Debug.Log(j + ": " + d.features[j]); } } dataPoints.Add(d); CheckIfEdgeData(d); count++; } } catch (IOException) { Debug.Log("Reading data interrupted at line " + i.ToString() + " at " + flag); } numPoints = numLines; // GUI progress: progress = i * 1.0f / (numLines - 1) * 1.0f; if (i % Mathf.FloorToInt(numLines / 20) == 0) { guiText = i.ToString() + " out of " + numLines.ToString() + " loaded!!"; //TODO removed yield will it break anything? } fish2 = sr.ReadLine().Trim(); buffer = fish2.Split(null); i += 1; } CalculateStdAndPercentiles(dataPoints); numLines = i; maxSizeTotal = new List <float>(topPercentile).ToArray(); currentDataPoints = new List <LabeledData>(dataPoints); }
/// <summary> /// Override only when overriding the normalization functions isn't enough. /// </summary> protected virtual LabeledData DataAsLabeledData(T[] data, L[] label) { double[] input = NormalizeInputData(data); double[] output = NormalizeOutputData(label); return(LabeledData.MakeFromArrays(input, output)); }
public void UnloadData() { Loaded = false; _Data = null; }