/// <summary> /// Calculates the Percentiles [0;1] of an Item. MaxEntriesInMemory is used to reduce the memory consumption by sweeping multiple times. Units is MB of memory /// </summary> public void Percentile(int Item, int[] TSteps, DFSBase df, double[] Percentiles, int MaxEntriesInMemory) { //List counts percentiles float[][] OutData = new float[Percentiles.Count()][]; for (int i = 0; i < Percentiles.Count(); i++) OutData[i] = new float[dfsdata.Count()]; List<int> steps = new List<int>(); steps.Add(0); //Get the delete values float delete = DfsDLLWrapper.dfsGetDeleteValFloat(_headerPointer); //Read first time step and create a list with the indeces of non-delete values ReadItemTimeStep(0, Item); List<int> NonDeleteEntries = new List<int>(); for (int i = 0; i < dfsdata.Length; i++) if (dfsdata[i] != delete) NonDeleteEntries.Add(i); //Find out how many sweeps are necessary to not exceed max memory double TotalData = (double)NonDeleteEntries.Count * (double)TSteps.Count(); if (TotalData > (MaxEntriesInMemory*40000)) { int nsteps = (int) Math.Max( TotalData / (MaxEntriesInMemory*40000),1); int StepLength = NonDeleteEntries.Count() / nsteps; for (int i = 0; i < nsteps; i++) steps.Add(steps.Last() + StepLength); } steps.Add(NonDeleteEntries.Count); //Now start the loop for (int m = 0; m < steps.Count-1; m++) { int dfscount = steps[m + 1] - steps[m]; //First iterater is dfsdata float[][] Data = new float[dfscount][]; for (int i = 0; i < Data.Count(); i++) Data[i] = new float[TSteps.Count()]; //Collect all data for (int i = 0; i < TSteps.Count(); i++) { var data = ReadItemTimeStep(TSteps[i], Item); int local = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { Data[local][i] = (dfsdata[NonDeleteEntries[k]]); local++; } } int local2 = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { //Convert to doubles from float double[] ddata = new double[TSteps.Count()]; for (int n = 0; n < TSteps.Count(); n++) ddata[n]=Data[local2][n]; //Calculate the percentile MathNet.Numerics.Statistics.Percentile pCalc = new MathNet.Numerics.Statistics.Percentile(ddata); pCalc.Method = MathNet.Numerics.Statistics.PercentileMethod.Excel; var p = pCalc.Compute(Percentiles); for (int l = 0; l < Percentiles.Count(); l++) OutData[l][NonDeleteEntries[k]] = (float)p[l]; local2++; } } //Insert deletevalues in output data for (int i = 0; i < dfsdata.Length; i++) { if (!NonDeleteEntries.Contains(i)) { for (int l = 0; l < Percentiles.Count(); l++) OutData[l][i] = delete; } } //Set Item infor for (int i = 0; i < Percentiles.Count(); i++) { df.Items[i].EumItem = Items[Item - 1].EumItem; df.Items[i].EumUnit = Items[Item - 1].EumUnit; df.Items[i].Name = Percentiles[i].ToString() + " percentile"; } for (int i = 0; i < Percentiles.Count(); i++) { df.WriteItemTimeStep(0, i + 1, OutData[i]); } }
/// <summary> /// Sums the values of the items to the selected time interval and puts them in the new dfs file /// Assumes that the there are delete values at the same places in all items and timesteps! /// </summary> /// <param name="Items"></param> /// <param name="df"></param> /// <param name="SumTim"></param> public void TimeAggregation(int[] Items, DFSBase df, TimeInterval SumTim, int Tsteps, MathType mathtype) { //Initialize all items and fill the buffer with Deletevalues Dictionary<int, float[]> BufferData = new Dictionary<int, float[]>(); foreach (var j in Items) { BufferData[j] = new float[dfsdata.Count()]; for (int i = 0; i < dfsdata.Count(); i++) BufferData[j][i] = (float)DeleteValue; } DateTime LastPrint = TimeSteps[0]; bool PrintNow = false; int tstepCounter = 0; //Loop the time steps for (int i = 0; i < NumberOfTimeSteps; i++) { tstepCounter++; switch (SumTim) { case TimeInterval.Year: PrintNow = (LastPrint.Year + Tsteps == TimeSteps[i].Year); break; case TimeInterval.Month: int nextmonth = LastPrint.Month + Tsteps; if (nextmonth > 12) nextmonth -= 12; PrintNow = (nextmonth == TimeSteps[i].Month); break; case TimeInterval.Day: PrintNow = ((TimeSteps[i].Subtract(LastPrint) >= TimeSpan.FromDays(Tsteps))); break; default: break; } //Now print summed values and empty buffer if (PrintNow) { foreach (var j in Items) { if (mathtype== MathType.Average) //Average, and a division with the number of time steps is required foreach (var n in NonDeleteIndex) BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter); if (df._timeAxis == TimeAxisType.CalendarNonEquidistant) df.AppendTimeStep(df.TimeSteps.Last().AddDays(30)); df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]); //Reset buffer foreach (var n in NonDeleteIndex) if (mathtype == MathType.Min) BufferData[j][n] = float.MaxValue; else if (mathtype == MathType.Max) BufferData[j][n] = float.MinValue; else BufferData[j][n] = 0; } tstepCounter = 0; LastPrint = TimeSteps[i]; PrintNow = false; } //Sum all items foreach (var j in Items) { ReadItemTimeStep(i, j); if (i == 0) //fill initial values in buffer { foreach (var k in NonDeleteIndex) { if (mathtype == MathType.Min) BufferData[j][k] = float.MaxValue; else if (mathtype == MathType.Max) BufferData[j][k] = float.MinValue; else BufferData[j][k] = 0; } } var arr = BufferData[j]; foreach (int k in NonDeleteIndex) if (mathtype == MathType.Min) arr[k] = Math.Min(arr[k], dfsdata[k]); else if (mathtype == MathType.Max) arr[k] = Math.Max(arr[k], dfsdata[k]); else arr[k] += dfsdata[k]; } } //print the last summed values foreach (var j in Items) { if (mathtype == MathType.Average) //If not sum it is average and a division with the number of time steps is required foreach (var n in NonDeleteIndex) BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter); df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]); } }
/// <summary> /// Sums the values of the items to the selected time interval and puts them in the new dfs file /// Assumes that the there are delete values at the same places in all items and timesteps! /// </summary> /// <param name="Items"></param> /// <param name="df"></param> /// <param name="SumTim"></param> public void MonthAggregation(int Item, DFSBase df) { SortedList<int, float[]> SumBuffer = new SortedList<int, float[]>(); SortedList<int, float[]> MaxBuffer = new SortedList<int, float[]>(); SortedList<int, float[]> MinBuffer = new SortedList<int, float[]>(); Dictionary<int, int> TstepCounter = new Dictionary<int, int>(); df.Items[0].Name = "MonthlyAverage"; df.items[0].EumItem = this.Items[Item-1].EumItem; df.Items[1].Name = "MonthlyMax"; df.items[1].EumItem = this.Items[Item-1].EumItem; df.Items[2].Name = "MonthlyMin"; df.items[2].EumItem = this.Items[Item-1].EumItem; //Initialize arrays foreach (var month in TimeSteps.Select(t => t.Month).Distinct()) { TstepCounter.Add(month, 0); SumBuffer.Add(month, new float[dfsdata.Count()]); MaxBuffer.Add(month, new float[dfsdata.Count()]); MinBuffer.Add(month, new float[dfsdata.Count()]); //Set everything to delete values for (int i = 0; i < dfsdata.Count(); i++) { SumBuffer[month][i] = (float)DeleteValue; MaxBuffer[month][i] = (float)DeleteValue; MinBuffer[month][i] = (float)DeleteValue; } ReadItemTimeStep(0, Item); foreach (var k in NonDeleteIndex) { SumBuffer[month][k] = 0; MaxBuffer[month][k] = float.MinValue; MinBuffer[month][k] = float.MaxValue; } } //Loop the time steps for (int i = 0; i < NumberOfTimeSteps; i++) { int currentmonth = TimeSteps[i].Month; ReadItemTimeStep(i, Item); TstepCounter[currentmonth] += 1; foreach (var k in NonDeleteIndex) { SumBuffer[currentmonth][k] += dfsdata[k]; if (dfsdata[k] != (float)DeleteValue) { MaxBuffer[currentmonth][k] = Math.Max(MaxBuffer[currentmonth][k], dfsdata[k]); MinBuffer[currentmonth][k] = Math.Min(MinBuffer[currentmonth][k], dfsdata[k]); } } } //Go from sum to average for(int i =0;i< SumBuffer.Count;i++) { foreach (var k in NonDeleteIndex) SumBuffer.Values[i][k] /= TstepCounter[SumBuffer.Keys[i]]; df.WriteItemTimeStep(i, 1, SumBuffer.Values[i]); df.WriteItemTimeStep(i, 2, MaxBuffer.Values[i]); df.WriteItemTimeStep(i, 3, MinBuffer.Values[i]); } }
/// <summary> /// Calculates the Percentiles [0;1] of an Item. MaxEntriesInMemory is used to reduce the memory consumption by sweeping multiple times. Units is MB of memory /// </summary> public void Percentile(int Item, int[] TSteps, DFSBase df, double[] Percentiles, int MaxEntriesInMemory) { //List counts percentiles float[][] OutData = new float[Percentiles.Count()][]; for (int i = 0; i < Percentiles.Count(); i++) { OutData[i] = new float[dfsdata.Count()]; } List <int> steps = new List <int>(); steps.Add(0); //Get the delete values float delete = DfsDLLWrapper.dfsGetDeleteValFloat(_headerPointer); //Read first time step and create a list with the indeces of non-delete values ReadItemTimeStep(0, Item); List <int> NonDeleteEntries = new List <int>(); for (int i = 0; i < dfsdata.Length; i++) { if (dfsdata[i] != delete) { NonDeleteEntries.Add(i); } } //Find out how many sweeps are necessary to not exceed max memory double TotalData = (double)NonDeleteEntries.Count * (double)TSteps.Count(); if (TotalData > (MaxEntriesInMemory * 40000)) { int nsteps = (int)Math.Max(TotalData / (MaxEntriesInMemory * 40000), 1); int StepLength = NonDeleteEntries.Count() / nsteps; for (int i = 0; i < nsteps; i++) { steps.Add(steps.Last() + StepLength); } } steps.Add(NonDeleteEntries.Count); //Now start the loop for (int m = 0; m < steps.Count - 1; m++) { int dfscount = steps[m + 1] - steps[m]; //First iterater is dfsdata float[][] Data = new float[dfscount][]; for (int i = 0; i < Data.Count(); i++) { Data[i] = new float[TSteps.Count()]; } //Collect all data for (int i = 0; i < TSteps.Count(); i++) { var data = ReadItemTimeStep(TSteps[i], Item); int local = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { Data[local][i] = (dfsdata[NonDeleteEntries[k]]); local++; } } int local2 = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { //Convert to doubles from float double[] ddata = new double[TSteps.Count()]; for (int n = 0; n < TSteps.Count(); n++) { ddata[n] = Data[local2][n]; } //Calculate the percentile MathNet.Numerics.Statistics.Percentile pCalc = new MathNet.Numerics.Statistics.Percentile(ddata); pCalc.Method = MathNet.Numerics.Statistics.PercentileMethod.Excel; var p = pCalc.Compute(Percentiles); for (int l = 0; l < Percentiles.Count(); l++) { OutData[l][NonDeleteEntries[k]] = (float)p[l]; } local2++; } } //Insert deletevalues in output data for (int i = 0; i < dfsdata.Length; i++) { if (!NonDeleteEntries.Contains(i)) { for (int l = 0; l < Percentiles.Count(); l++) { OutData[l][i] = delete; } } } //Set Item infor for (int i = 0; i < Percentiles.Count(); i++) { df.Items[i].EumItem = Items[Item - 1].EumItem; df.Items[i].EumUnit = Items[Item - 1].EumUnit; df.Items[i].Name = Percentiles[i].ToString() + " percentile"; } for (int i = 0; i < Percentiles.Count(); i++) { df.WriteItemTimeStep(0, i + 1, OutData[i]); } }
/// <summary> /// Sums the values of the items to the selected time interval and puts them in the new dfs file /// Assumes that the there are delete values at the same places in all items and timesteps! /// </summary> /// <param name="Items"></param> /// <param name="df"></param> /// <param name="SumTim"></param> public void TimeAggregation(int[] Items, DFSBase df, TimeInterval SumTim, int Tsteps, MathType mathtype) { //Initialize all items and fill the buffer with Deletevalues Dictionary <int, float[]> BufferData = new Dictionary <int, float[]>(); foreach (var j in Items) { BufferData[j] = new float[dfsdata.Count()]; for (int i = 0; i < dfsdata.Count(); i++) { BufferData[j][i] = (float)DeleteValue; } } DateTime LastPrint = TimeSteps[0]; bool PrintNow = false; int tstepCounter = 0; //Loop the time steps for (int i = 0; i < NumberOfTimeSteps; i++) { tstepCounter++; switch (SumTim) { case TimeInterval.Year: PrintNow = (LastPrint.Year + Tsteps == TimeSteps[i].Year); break; case TimeInterval.Month: int nextmonth = LastPrint.Month + Tsteps; if (nextmonth > 12) { nextmonth -= 12; } PrintNow = (nextmonth == TimeSteps[i].Month); break; case TimeInterval.Day: PrintNow = ((TimeSteps[i].Subtract(LastPrint) >= TimeSpan.FromDays(Tsteps))); break; default: break; } //Now print summed values and empty buffer if (PrintNow) { foreach (var j in Items) { if (mathtype == MathType.Average) //Average, and a division with the number of time steps is required { foreach (var n in NonDeleteIndex) { BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter); } } if (df._timeAxis == TimeAxisType.CalendarNonEquidistant) { df.AppendTimeStep(df.TimeSteps.Last().AddDays(30)); } df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]); //Reset buffer foreach (var n in NonDeleteIndex) { if (mathtype == MathType.Min) { BufferData[j][n] = float.MaxValue; } else if (mathtype == MathType.Max) { BufferData[j][n] = float.MinValue; } else { BufferData[j][n] = 0; } } } tstepCounter = 0; LastPrint = TimeSteps[i]; PrintNow = false; } //Sum all items foreach (var j in Items) { ReadItemTimeStep(i, j); if (i == 0) //fill initial values in buffer { foreach (var k in NonDeleteIndex) { if (mathtype == MathType.Min) { BufferData[j][k] = float.MaxValue; } else if (mathtype == MathType.Max) { BufferData[j][k] = float.MinValue; } else { BufferData[j][k] = 0; } } } var arr = BufferData[j]; foreach (int k in NonDeleteIndex) { if (mathtype == MathType.Min) { arr[k] = Math.Min(arr[k], dfsdata[k]); } else if (mathtype == MathType.Max) { arr[k] = Math.Max(arr[k], dfsdata[k]); } else { arr[k] += dfsdata[k]; } } } } //print the last summed values foreach (var j in Items) { if (mathtype == MathType.Average) //If not sum it is average and a division with the number of time steps is required { foreach (var n in NonDeleteIndex) { BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter); } } df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]); } }
/// <summary> /// Sums the values of the items to the selected time interval and puts them in the new dfs file /// Assumes that the there are delete values at the same places in all items and timesteps! /// </summary> /// <param name="Items"></param> /// <param name="df"></param> /// <param name="SumTim"></param> public void MonthAggregation(int Item, DFSBase df) { SortedList <int, float[]> SumBuffer = new SortedList <int, float[]>(); SortedList <int, float[]> MaxBuffer = new SortedList <int, float[]>(); SortedList <int, float[]> MinBuffer = new SortedList <int, float[]>(); Dictionary <int, int> TstepCounter = new Dictionary <int, int>(); df.Items[0].Name = "MonthlyAverage"; df.items[0].EumItem = this.Items[Item - 1].EumItem; df.Items[1].Name = "MonthlyMax"; df.items[1].EumItem = this.Items[Item - 1].EumItem; df.Items[2].Name = "MonthlyMin"; df.items[2].EumItem = this.Items[Item - 1].EumItem; //Initialize arrays foreach (var month in TimeSteps.Select(t => t.Month).Distinct()) { TstepCounter.Add(month, 0); SumBuffer.Add(month, new float[dfsdata.Count()]); MaxBuffer.Add(month, new float[dfsdata.Count()]); MinBuffer.Add(month, new float[dfsdata.Count()]); //Set everything to delete values for (int i = 0; i < dfsdata.Count(); i++) { SumBuffer[month][i] = (float)DeleteValue; MaxBuffer[month][i] = (float)DeleteValue; MinBuffer[month][i] = (float)DeleteValue; } ReadItemTimeStep(0, Item); foreach (var k in NonDeleteIndex) { SumBuffer[month][k] = 0; MaxBuffer[month][k] = float.MinValue; MinBuffer[month][k] = float.MaxValue; } } //Loop the time steps for (int i = 0; i < NumberOfTimeSteps; i++) { int currentmonth = TimeSteps[i].Month; ReadItemTimeStep(i, Item); TstepCounter[currentmonth] += 1; foreach (var k in NonDeleteIndex) { SumBuffer[currentmonth][k] += dfsdata[k]; if (dfsdata[k] != (float)DeleteValue) { MaxBuffer[currentmonth][k] = Math.Max(MaxBuffer[currentmonth][k], dfsdata[k]); MinBuffer[currentmonth][k] = Math.Min(MinBuffer[currentmonth][k], dfsdata[k]); } } } //Go from sum to average for (int i = 0; i < SumBuffer.Count; i++) { foreach (var k in NonDeleteIndex) { SumBuffer.Values[i][k] /= TstepCounter[SumBuffer.Keys[i]]; } df.WriteItemTimeStep(i, 1, SumBuffer.Values[i]); df.WriteItemTimeStep(i, 2, MaxBuffer.Values[i]); df.WriteItemTimeStep(i, 3, MinBuffer.Values[i]); } }