/// <summary>
/// Calculates the Percentiles [0;1] of an Item. MaxEntriesInMemory is used to reduce the memory consumption by sweeping multiple times. Units is MB of memory
/// </summary>
public void Percentile(int Item, int[] TSteps, DFSBase df, double[] Percentiles, int MaxEntriesInMemory)
{
//List counts percentiles
float[][] OutData = new float[Percentiles.Count()][];
for (int i = 0; i < Percentiles.Count(); i++)
OutData[i] = new float[dfsdata.Count()];
List<int> steps = new List<int>();
steps.Add(0);
//Get the delete values
float delete = DfsDLLWrapper.dfsGetDeleteValFloat(_headerPointer);
//Read first time step and create a list with the indeces of non-delete values
ReadItemTimeStep(0, Item);
List<int> NonDeleteEntries = new List<int>();
for (int i = 0; i < dfsdata.Length; i++)
if (dfsdata[i] != delete)
NonDeleteEntries.Add(i);
//Find out how many sweeps are necessary to not exceed max memory
double TotalData = (double)NonDeleteEntries.Count * (double)TSteps.Count();
if (TotalData > (MaxEntriesInMemory*40000))
{
int nsteps = (int) Math.Max( TotalData / (MaxEntriesInMemory*40000),1);
int StepLength = NonDeleteEntries.Count() / nsteps;
for (int i = 0; i < nsteps; i++)
steps.Add(steps.Last() + StepLength);
}
steps.Add(NonDeleteEntries.Count);
//Now start the loop
for (int m = 0; m < steps.Count-1; m++)
{
int dfscount = steps[m + 1] - steps[m];
//First iterater is dfsdata
float[][] Data = new float[dfscount][];
for (int i = 0; i < Data.Count(); i++)
Data[i] = new float[TSteps.Count()];
//Collect all data
for (int i = 0; i < TSteps.Count(); i++)
{
var data = ReadItemTimeStep(TSteps[i], Item);
int local = 0;
for (int k = steps[m]; k < steps[m + 1]; k++)
{
Data[local][i] = (dfsdata[NonDeleteEntries[k]]);
local++;
}
}
int local2 = 0;
for (int k = steps[m]; k < steps[m + 1]; k++)
{
//Convert to doubles from float
double[] ddata = new double[TSteps.Count()];
for (int n = 0; n < TSteps.Count(); n++)
ddata[n]=Data[local2][n];
//Calculate the percentile
MathNet.Numerics.Statistics.Percentile pCalc = new MathNet.Numerics.Statistics.Percentile(ddata);
pCalc.Method = MathNet.Numerics.Statistics.PercentileMethod.Excel;
var p = pCalc.Compute(Percentiles);
for (int l = 0; l < Percentiles.Count(); l++)
OutData[l][NonDeleteEntries[k]] = (float)p[l];
local2++;
}
}
//Insert deletevalues in output data
for (int i = 0; i < dfsdata.Length; i++)
{
if (!NonDeleteEntries.Contains(i))
{
for (int l = 0; l < Percentiles.Count(); l++)
OutData[l][i] = delete;
}
}
//Set Item infor
for (int i = 0; i < Percentiles.Count(); i++)
{
df.Items[i].EumItem = Items[Item - 1].EumItem;
df.Items[i].EumUnit = Items[Item - 1].EumUnit;
df.Items[i].Name = Percentiles[i].ToString() + " percentile";
}
for (int i = 0; i < Percentiles.Count(); i++)
{
df.WriteItemTimeStep(0, i + 1, OutData[i]);
}
}
/// <summary>
/// Sums the values of the items to the selected time interval and puts them in the new dfs file
/// Assumes that the there are delete values at the same places in all items and timesteps!
/// </summary>
/// <param name="Items"></param>
/// <param name="df"></param>
/// <param name="SumTim"></param>
public void TimeAggregation(int[] Items, DFSBase df, TimeInterval SumTim, int Tsteps, MathType mathtype)
{
//Initialize all items and fill the buffer with Deletevalues
Dictionary<int, float[]> BufferData = new Dictionary<int, float[]>();
foreach (var j in Items)
{
BufferData[j] = new float[dfsdata.Count()];
for (int i = 0; i < dfsdata.Count(); i++)
BufferData[j][i] = (float)DeleteValue;
}
DateTime LastPrint = TimeSteps[0];
bool PrintNow = false;
int tstepCounter = 0;
//Loop the time steps
for (int i = 0; i < NumberOfTimeSteps; i++)
{
tstepCounter++;
switch (SumTim)
{
case TimeInterval.Year:
PrintNow = (LastPrint.Year + Tsteps == TimeSteps[i].Year);
break;
case TimeInterval.Month:
int nextmonth = LastPrint.Month + Tsteps;
if (nextmonth > 12)
nextmonth -= 12;
PrintNow = (nextmonth == TimeSteps[i].Month);
break;
case TimeInterval.Day:
PrintNow = ((TimeSteps[i].Subtract(LastPrint) >= TimeSpan.FromDays(Tsteps)));
break;
default:
break;
}
//Now print summed values and empty buffer
if (PrintNow)
{
foreach (var j in Items)
{
if (mathtype== MathType.Average) //Average, and a division with the number of time steps is required
foreach (var n in NonDeleteIndex)
BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter);
if (df._timeAxis == TimeAxisType.CalendarNonEquidistant)
df.AppendTimeStep(df.TimeSteps.Last().AddDays(30));
df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]);
//Reset buffer
foreach (var n in NonDeleteIndex)
if (mathtype == MathType.Min)
BufferData[j][n] = float.MaxValue;
else if (mathtype == MathType.Max)
BufferData[j][n] = float.MinValue;
else
BufferData[j][n] = 0;
}
tstepCounter = 0;
LastPrint = TimeSteps[i];
PrintNow = false;
}
//Sum all items
foreach (var j in Items)
{
ReadItemTimeStep(i, j);
if (i == 0) //fill initial values in buffer
{
foreach (var k in NonDeleteIndex)
{
if (mathtype == MathType.Min)
BufferData[j][k] = float.MaxValue;
else if (mathtype == MathType.Max)
BufferData[j][k] = float.MinValue;
else
BufferData[j][k] = 0;
}
}
var arr = BufferData[j];
foreach (int k in NonDeleteIndex)
if (mathtype == MathType.Min)
arr[k] = Math.Min(arr[k], dfsdata[k]);
else if (mathtype == MathType.Max)
arr[k] = Math.Max(arr[k], dfsdata[k]);
else
arr[k] += dfsdata[k];
}
}
//print the last summed values
foreach (var j in Items)
{
if (mathtype == MathType.Average) //If not sum it is average and a division with the number of time steps is required
foreach (var n in NonDeleteIndex)
BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter);
df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]);
}
}
/// <summary>
/// Sums the values of the items to the selected time interval and puts them in the new dfs file
/// Assumes that the there are delete values at the same places in all items and timesteps!
/// </summary>
/// <param name="Items"></param>
/// <param name="df"></param>
/// <param name="SumTim"></param>
public void MonthAggregation(int Item, DFSBase df)
{
SortedList<int, float[]> SumBuffer = new SortedList<int, float[]>();
SortedList<int, float[]> MaxBuffer = new SortedList<int, float[]>();
SortedList<int, float[]> MinBuffer = new SortedList<int, float[]>();
Dictionary<int, int> TstepCounter = new Dictionary<int, int>();
df.Items[0].Name = "MonthlyAverage";
df.items[0].EumItem = this.Items[Item-1].EumItem;
df.Items[1].Name = "MonthlyMax";
df.items[1].EumItem = this.Items[Item-1].EumItem;
df.Items[2].Name = "MonthlyMin";
df.items[2].EumItem = this.Items[Item-1].EumItem;
//Initialize arrays
foreach (var month in TimeSteps.Select(t => t.Month).Distinct())
{
TstepCounter.Add(month, 0);
SumBuffer.Add(month, new float[dfsdata.Count()]);
MaxBuffer.Add(month, new float[dfsdata.Count()]);
MinBuffer.Add(month, new float[dfsdata.Count()]);
//Set everything to delete values
for (int i = 0; i < dfsdata.Count(); i++)
{
SumBuffer[month][i] = (float)DeleteValue;
MaxBuffer[month][i] = (float)DeleteValue;
MinBuffer[month][i] = (float)DeleteValue;
}
ReadItemTimeStep(0, Item);
foreach (var k in NonDeleteIndex)
{
SumBuffer[month][k] = 0;
MaxBuffer[month][k] = float.MinValue;
MinBuffer[month][k] = float.MaxValue;
}
}
//Loop the time steps
for (int i = 0; i < NumberOfTimeSteps; i++)
{
int currentmonth = TimeSteps[i].Month;
ReadItemTimeStep(i, Item);
TstepCounter[currentmonth] += 1;
foreach (var k in NonDeleteIndex)
{
SumBuffer[currentmonth][k] += dfsdata[k];
if (dfsdata[k] != (float)DeleteValue)
{
MaxBuffer[currentmonth][k] = Math.Max(MaxBuffer[currentmonth][k], dfsdata[k]);
MinBuffer[currentmonth][k] = Math.Min(MinBuffer[currentmonth][k], dfsdata[k]);
}
}
}
//Go from sum to average
for(int i =0;i< SumBuffer.Count;i++)
{
foreach (var k in NonDeleteIndex)
SumBuffer.Values[i][k] /= TstepCounter[SumBuffer.Keys[i]];
df.WriteItemTimeStep(i, 1, SumBuffer.Values[i]);
df.WriteItemTimeStep(i, 2, MaxBuffer.Values[i]);
df.WriteItemTimeStep(i, 3, MinBuffer.Values[i]);
}
}
/// <summary>
/// Calculates the Percentiles [0;1] of an Item. MaxEntriesInMemory is used to reduce the memory consumption by sweeping multiple times. Units is MB of memory
/// </summary>
public void Percentile(int Item, int[] TSteps, DFSBase df, double[] Percentiles, int MaxEntriesInMemory)
{
//List counts percentiles
float[][] OutData = new float[Percentiles.Count()][];
for (int i = 0; i < Percentiles.Count(); i++)
{
OutData[i] = new float[dfsdata.Count()];
}
List <int> steps = new List <int>();
steps.Add(0);
//Get the delete values
float delete = DfsDLLWrapper.dfsGetDeleteValFloat(_headerPointer);
//Read first time step and create a list with the indeces of non-delete values
ReadItemTimeStep(0, Item);
List <int> NonDeleteEntries = new List <int>();
for (int i = 0; i < dfsdata.Length; i++)
{
if (dfsdata[i] != delete)
{
NonDeleteEntries.Add(i);
}
}
//Find out how many sweeps are necessary to not exceed max memory
double TotalData = (double)NonDeleteEntries.Count * (double)TSteps.Count();
if (TotalData > (MaxEntriesInMemory * 40000))
{
int nsteps = (int)Math.Max(TotalData / (MaxEntriesInMemory * 40000), 1);
int StepLength = NonDeleteEntries.Count() / nsteps;
for (int i = 0; i < nsteps; i++)
{
steps.Add(steps.Last() + StepLength);
}
}
steps.Add(NonDeleteEntries.Count);
//Now start the loop
for (int m = 0; m < steps.Count - 1; m++)
{
int dfscount = steps[m + 1] - steps[m];
//First iterater is dfsdata
float[][] Data = new float[dfscount][];
for (int i = 0; i < Data.Count(); i++)
{
Data[i] = new float[TSteps.Count()];
}
//Collect all data
for (int i = 0; i < TSteps.Count(); i++)
{
var data = ReadItemTimeStep(TSteps[i], Item);
int local = 0;
for (int k = steps[m]; k < steps[m + 1]; k++)
{
Data[local][i] = (dfsdata[NonDeleteEntries[k]]);
local++;
}
}
int local2 = 0;
for (int k = steps[m]; k < steps[m + 1]; k++)
{
//Convert to doubles from float
double[] ddata = new double[TSteps.Count()];
for (int n = 0; n < TSteps.Count(); n++)
{
ddata[n] = Data[local2][n];
}
//Calculate the percentile
MathNet.Numerics.Statistics.Percentile pCalc = new MathNet.Numerics.Statistics.Percentile(ddata);
pCalc.Method = MathNet.Numerics.Statistics.PercentileMethod.Excel;
var p = pCalc.Compute(Percentiles);
for (int l = 0; l < Percentiles.Count(); l++)
{
OutData[l][NonDeleteEntries[k]] = (float)p[l];
}
local2++;
}
}
//Insert deletevalues in output data
for (int i = 0; i < dfsdata.Length; i++)
{
if (!NonDeleteEntries.Contains(i))
{
for (int l = 0; l < Percentiles.Count(); l++)
{
OutData[l][i] = delete;
}
}
}
//Set Item infor
for (int i = 0; i < Percentiles.Count(); i++)
{
df.Items[i].EumItem = Items[Item - 1].EumItem;
df.Items[i].EumUnit = Items[Item - 1].EumUnit;
df.Items[i].Name = Percentiles[i].ToString() + " percentile";
}
for (int i = 0; i < Percentiles.Count(); i++)
{
df.WriteItemTimeStep(0, i + 1, OutData[i]);
}
}
/// <summary>
/// Sums the values of the items to the selected time interval and puts them in the new dfs file
/// Assumes that the there are delete values at the same places in all items and timesteps!
/// </summary>
/// <param name="Items"></param>
/// <param name="df"></param>
/// <param name="SumTim"></param>
public void TimeAggregation(int[] Items, DFSBase df, TimeInterval SumTim, int Tsteps, MathType mathtype)
{
//Initialize all items and fill the buffer with Deletevalues
Dictionary <int, float[]> BufferData = new Dictionary <int, float[]>();
foreach (var j in Items)
{
BufferData[j] = new float[dfsdata.Count()];
for (int i = 0; i < dfsdata.Count(); i++)
{
BufferData[j][i] = (float)DeleteValue;
}
}
DateTime LastPrint = TimeSteps[0];
bool PrintNow = false;
int tstepCounter = 0;
//Loop the time steps
for (int i = 0; i < NumberOfTimeSteps; i++)
{
tstepCounter++;
switch (SumTim)
{
case TimeInterval.Year:
PrintNow = (LastPrint.Year + Tsteps == TimeSteps[i].Year);
break;
case TimeInterval.Month:
int nextmonth = LastPrint.Month + Tsteps;
if (nextmonth > 12)
{
nextmonth -= 12;
}
PrintNow = (nextmonth == TimeSteps[i].Month);
break;
case TimeInterval.Day:
PrintNow = ((TimeSteps[i].Subtract(LastPrint) >= TimeSpan.FromDays(Tsteps)));
break;
default:
break;
}
//Now print summed values and empty buffer
if (PrintNow)
{
foreach (var j in Items)
{
if (mathtype == MathType.Average) //Average, and a division with the number of time steps is required
{
foreach (var n in NonDeleteIndex)
{
BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter);
}
}
if (df._timeAxis == TimeAxisType.CalendarNonEquidistant)
{
df.AppendTimeStep(df.TimeSteps.Last().AddDays(30));
}
df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]);
//Reset buffer
foreach (var n in NonDeleteIndex)
{
if (mathtype == MathType.Min)
{
BufferData[j][n] = float.MaxValue;
}
else if (mathtype == MathType.Max)
{
BufferData[j][n] = float.MinValue;
}
else
{
BufferData[j][n] = 0;
}
}
}
tstepCounter = 0;
LastPrint = TimeSteps[i];
PrintNow = false;
}
//Sum all items
foreach (var j in Items)
{
ReadItemTimeStep(i, j);
if (i == 0) //fill initial values in buffer
{
foreach (var k in NonDeleteIndex)
{
if (mathtype == MathType.Min)
{
BufferData[j][k] = float.MaxValue;
}
else if (mathtype == MathType.Max)
{
BufferData[j][k] = float.MinValue;
}
else
{
BufferData[j][k] = 0;
}
}
}
var arr = BufferData[j];
foreach (int k in NonDeleteIndex)
{
if (mathtype == MathType.Min)
{
arr[k] = Math.Min(arr[k], dfsdata[k]);
}
else if (mathtype == MathType.Max)
{
arr[k] = Math.Max(arr[k], dfsdata[k]);
}
else
{
arr[k] += dfsdata[k];
}
}
}
}
//print the last summed values
foreach (var j in Items)
{
if (mathtype == MathType.Average) //If not sum it is average and a division with the number of time steps is required
{
foreach (var n in NonDeleteIndex)
{
BufferData[j][n] = BufferData[j][n] / ((float)tstepCounter);
}
}
df.WriteItemTimeStep(df.NumberOfTimeSteps, j, BufferData[j]);
}
}
/// <summary>
/// Sums the values of the items to the selected time interval and puts them in the new dfs file
/// Assumes that the there are delete values at the same places in all items and timesteps!
/// </summary>
/// <param name="Items"></param>
/// <param name="df"></param>
/// <param name="SumTim"></param>
public void MonthAggregation(int Item, DFSBase df)
{
SortedList <int, float[]> SumBuffer = new SortedList <int, float[]>();
SortedList <int, float[]> MaxBuffer = new SortedList <int, float[]>();
SortedList <int, float[]> MinBuffer = new SortedList <int, float[]>();
Dictionary <int, int> TstepCounter = new Dictionary <int, int>();
df.Items[0].Name = "MonthlyAverage";
df.items[0].EumItem = this.Items[Item - 1].EumItem;
df.Items[1].Name = "MonthlyMax";
df.items[1].EumItem = this.Items[Item - 1].EumItem;
df.Items[2].Name = "MonthlyMin";
df.items[2].EumItem = this.Items[Item - 1].EumItem;
//Initialize arrays
foreach (var month in TimeSteps.Select(t => t.Month).Distinct())
{
TstepCounter.Add(month, 0);
SumBuffer.Add(month, new float[dfsdata.Count()]);
MaxBuffer.Add(month, new float[dfsdata.Count()]);
MinBuffer.Add(month, new float[dfsdata.Count()]);
//Set everything to delete values
for (int i = 0; i < dfsdata.Count(); i++)
{
SumBuffer[month][i] = (float)DeleteValue;
MaxBuffer[month][i] = (float)DeleteValue;
MinBuffer[month][i] = (float)DeleteValue;
}
ReadItemTimeStep(0, Item);
foreach (var k in NonDeleteIndex)
{
SumBuffer[month][k] = 0;
MaxBuffer[month][k] = float.MinValue;
MinBuffer[month][k] = float.MaxValue;
}
}
//Loop the time steps
for (int i = 0; i < NumberOfTimeSteps; i++)
{
int currentmonth = TimeSteps[i].Month;
ReadItemTimeStep(i, Item);
TstepCounter[currentmonth] += 1;
foreach (var k in NonDeleteIndex)
{
SumBuffer[currentmonth][k] += dfsdata[k];
if (dfsdata[k] != (float)DeleteValue)
{
MaxBuffer[currentmonth][k] = Math.Max(MaxBuffer[currentmonth][k], dfsdata[k]);
MinBuffer[currentmonth][k] = Math.Min(MinBuffer[currentmonth][k], dfsdata[k]);
}
}
}
//Go from sum to average
for (int i = 0; i < SumBuffer.Count; i++)
{
foreach (var k in NonDeleteIndex)
{
SumBuffer.Values[i][k] /= TstepCounter[SumBuffer.Keys[i]];
}
df.WriteItemTimeStep(i, 1, SumBuffer.Values[i]);
df.WriteItemTimeStep(i, 2, MaxBuffer.Values[i]);
df.WriteItemTimeStep(i, 3, MinBuffer.Values[i]);
}
}
