/// <summary> /// Calculates the Percentiles [0;1] of an Item. MaxEntriesInMemory is used to reduce the memory consumption by sweeping multiple times. Units is MB of memory /// </summary> public void Percentile(int Item, int[] TSteps, DFSBase df, double[] Percentiles, int MaxEntriesInMemory) { //List counts percentiles float[][] OutData = new float[Percentiles.Count()][]; for (int i = 0; i < Percentiles.Count(); i++) OutData[i] = new float[dfsdata.Count()]; List<int> steps = new List<int>(); steps.Add(0); //Get the delete values float delete = DfsDLLWrapper.dfsGetDeleteValFloat(_headerPointer); //Read first time step and create a list with the indeces of non-delete values ReadItemTimeStep(0, Item); List<int> NonDeleteEntries = new List<int>(); for (int i = 0; i < dfsdata.Length; i++) if (dfsdata[i] != delete) NonDeleteEntries.Add(i); //Find out how many sweeps are necessary to not exceed max memory double TotalData = (double)NonDeleteEntries.Count * (double)TSteps.Count(); if (TotalData > (MaxEntriesInMemory*40000)) { int nsteps = (int) Math.Max( TotalData / (MaxEntriesInMemory*40000),1); int StepLength = NonDeleteEntries.Count() / nsteps; for (int i = 0; i < nsteps; i++) steps.Add(steps.Last() + StepLength); } steps.Add(NonDeleteEntries.Count); //Now start the loop for (int m = 0; m < steps.Count-1; m++) { int dfscount = steps[m + 1] - steps[m]; //First iterater is dfsdata float[][] Data = new float[dfscount][]; for (int i = 0; i < Data.Count(); i++) Data[i] = new float[TSteps.Count()]; //Collect all data for (int i = 0; i < TSteps.Count(); i++) { var data = ReadItemTimeStep(TSteps[i], Item); int local = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { Data[local][i] = (dfsdata[NonDeleteEntries[k]]); local++; } } int local2 = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { //Convert to doubles from float double[] ddata = new double[TSteps.Count()]; for (int n = 0; n < TSteps.Count(); n++) ddata[n]=Data[local2][n]; //Calculate the percentile MathNet.Numerics.Statistics.Percentile pCalc = new MathNet.Numerics.Statistics.Percentile(ddata); pCalc.Method = MathNet.Numerics.Statistics.PercentileMethod.Excel; var p = pCalc.Compute(Percentiles); for (int l = 0; l < Percentiles.Count(); l++) OutData[l][NonDeleteEntries[k]] = (float)p[l]; local2++; } } //Insert deletevalues in output data for (int i = 0; i < dfsdata.Length; i++) { if (!NonDeleteEntries.Contains(i)) { for (int l = 0; l < Percentiles.Count(); l++) OutData[l][i] = delete; } } //Set Item infor for (int i = 0; i < Percentiles.Count(); i++) { df.Items[i].EumItem = Items[Item - 1].EumItem; df.Items[i].EumUnit = Items[Item - 1].EumUnit; df.Items[i].Name = Percentiles[i].ToString() + " percentile"; } for (int i = 0; i < Percentiles.Count(); i++) { df.WriteItemTimeStep(0, i + 1, OutData[i]); } }
public void DebugPrint(string Directory) { //We need to process data for extra output while we have the particles { foreach (var c in Catchments.Where(ca => ca.EndParticles.Count >= 1)) { if (c.EndParticles.Count >= 20) { c.ParticleBreakthroughCurves = new List <Tuple <double, double> >(); MathNet.Numerics.Statistics.Percentile p = new MathNet.Numerics.Statistics.Percentile(c.EndParticles.Select(pa => pa.TravelTime)); for (int i = 1; i < np; i++) { c.ParticleBreakthroughCurves.Add(new Tuple <double, double>(i / np * 100.0, p.Compute(i / np))); } //Also do oxidized breakthrough curves if (c.EndParticles.Count(pp => pp.Registration != 1) >= 20) { c.ParticleBreakthroughCurvesOxidized = new List <Tuple <double, double> >(); p = new MathNet.Numerics.Statistics.Percentile(c.EndParticles.Where(pp => pp.Registration != 1).Select(pa => pa.TravelTime)); for (int i = 1; i < np; i++) { c.ParticleBreakthroughCurvesOxidized.Add(new Tuple <double, double>(i / np * 100.0, p.Compute(i / np))); } } } DataRow row = DebugData.Rows.Find(c.ID); if (row == null) { row = DebugData.NewRow(); row[0] = c.ID; DebugData.Rows.Add(row); } row["PartCount"] = c.EndParticles.Count; row["RedoxCount"] = c.EndParticles.Count(pp => pp.Registration == 1); row["RedoxRatio"] = c.EndParticles.Count(pp => pp.Registration == 1) / (double)c.EndParticles.Count; if (c.EndParticles.Count > 0) { row["Drain_to_River"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.Drain_to_River) / (double)c.EndParticles.Count; row["Drain_to_Boundary"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.Drain_to_Boundary) / (double)c.EndParticles.Count; row["Unsaturated_zone"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.Unsaturated_zone) / (double)c.EndParticles.Count; row["River"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.River) / (double)c.EndParticles.Count; } row["PartCount_start"] = c.StartParticles.Count; } } NewMessage("Writing breakthrough curves"); var selectedCatchments = Catchments.Where(cc => cc.ParticleBreakthroughCurves != null); using (System.IO.StreamWriter sw = new System.IO.StreamWriter(Path.Combine(Directory, "BC.csv"))) { StringBuilder headline = new StringBuilder(); headline.Append("ID\tNumber of Particles"); for (int i = 1; i < np; i++) { headline.Append("\t + " + (i / np * 100.0)); } sw.WriteLine(headline); foreach (var c in selectedCatchments.Where(cc => cc.ParticleBreakthroughCurves != null)) { StringBuilder line = new StringBuilder(); line.Append(c.ID + "\t" + c.EndParticles.Count); foreach (var pe in c.ParticleBreakthroughCurves) { line.Append("\t" + pe.Item2); } sw.WriteLine(line); } } if (selectedCatchments.Count() > 0) { using (ShapeWriter sw = new ShapeWriter(Path.Combine(Directory, Name + "_debug.shp")) { Projection = MainModel.projection }) { foreach (var bc in selectedCatchments.First().ParticleBreakthroughCurves) { DebugData.Columns.Add(((int)bc.Item1).ToString(), typeof(double)); } foreach (var bc in selectedCatchments.First().ParticleBreakthroughCurves) { DebugData.Columns.Add(((int)bc.Item1).ToString() + "Ox", typeof(double)); } foreach (var c in selectedCatchments) { GeoRefData gd = new GeoRefData() { Geometry = c.Geometry }; var row = DebugData.Rows.Find(c.ID); if (c.ParticleBreakthroughCurves != null) { foreach (var bc in c.ParticleBreakthroughCurves) { row[((int)bc.Item1).ToString()] = bc.Item2; } } if (c.ParticleBreakthroughCurvesOxidized != null) { foreach (var bc in c.ParticleBreakthroughCurvesOxidized) { row[((int)bc.Item1).ToString() + "Ox"] = bc.Item2; } } gd.Data = row; sw.Write(gd); } } } //selectedCatchments = Catchments.Where(cc => cc.EndParticles.Count > 0).ToList(); //foreach (var c in selectedCatchments) //{ // DataTable dt = new DataTable(); // dt.Columns.Add("Part_Id", typeof(int)); // dt.Columns.Add("Sink", typeof(string)); // dt.Columns.Add("Reg", typeof(int)); // using (ShapeWriter sw = new ShapeWriter(Path.Combine(Directory, c.ID + "_particles.shp")) { Projection = MainModel.projection }) // { // foreach (var p in c.EndParticles) // { // var row = dt.NewRow(); // row["Part_Id"] = p.ID; // row["Sink"] = p.SinkType.ToString(); // row["Reg"] = p.Registration; // sw.Write(new GeoRefData() { Geometry = new XYLine(p.XStart, p.YStart, p.X, p.Y), Data = row }); // } // } //} }
/// <summary> /// Calculates the Percentiles [0;1] of an Item. MaxEntriesInMemory is used to reduce the memory consumption by sweeping multiple times. Units is MB of memory /// </summary> public void Percentile(int Item, int[] TSteps, DFSBase df, double[] Percentiles, int MaxEntriesInMemory) { //List counts percentiles float[][] OutData = new float[Percentiles.Count()][]; for (int i = 0; i < Percentiles.Count(); i++) { OutData[i] = new float[dfsdata.Count()]; } List <int> steps = new List <int>(); steps.Add(0); //Get the delete values float delete = DfsDLLWrapper.dfsGetDeleteValFloat(_headerPointer); //Read first time step and create a list with the indeces of non-delete values ReadItemTimeStep(0, Item); List <int> NonDeleteEntries = new List <int>(); for (int i = 0; i < dfsdata.Length; i++) { if (dfsdata[i] != delete) { NonDeleteEntries.Add(i); } } //Find out how many sweeps are necessary to not exceed max memory double TotalData = (double)NonDeleteEntries.Count * (double)TSteps.Count(); if (TotalData > (MaxEntriesInMemory * 40000)) { int nsteps = (int)Math.Max(TotalData / (MaxEntriesInMemory * 40000), 1); int StepLength = NonDeleteEntries.Count() / nsteps; for (int i = 0; i < nsteps; i++) { steps.Add(steps.Last() + StepLength); } } steps.Add(NonDeleteEntries.Count); //Now start the loop for (int m = 0; m < steps.Count - 1; m++) { int dfscount = steps[m + 1] - steps[m]; //First iterater is dfsdata float[][] Data = new float[dfscount][]; for (int i = 0; i < Data.Count(); i++) { Data[i] = new float[TSteps.Count()]; } //Collect all data for (int i = 0; i < TSteps.Count(); i++) { var data = ReadItemTimeStep(TSteps[i], Item); int local = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { Data[local][i] = (dfsdata[NonDeleteEntries[k]]); local++; } } int local2 = 0; for (int k = steps[m]; k < steps[m + 1]; k++) { //Convert to doubles from float double[] ddata = new double[TSteps.Count()]; for (int n = 0; n < TSteps.Count(); n++) { ddata[n] = Data[local2][n]; } //Calculate the percentile MathNet.Numerics.Statistics.Percentile pCalc = new MathNet.Numerics.Statistics.Percentile(ddata); pCalc.Method = MathNet.Numerics.Statistics.PercentileMethod.Excel; var p = pCalc.Compute(Percentiles); for (int l = 0; l < Percentiles.Count(); l++) { OutData[l][NonDeleteEntries[k]] = (float)p[l]; } local2++; } } //Insert deletevalues in output data for (int i = 0; i < dfsdata.Length; i++) { if (!NonDeleteEntries.Contains(i)) { for (int l = 0; l < Percentiles.Count(); l++) { OutData[l][i] = delete; } } } //Set Item infor for (int i = 0; i < Percentiles.Count(); i++) { df.Items[i].EumItem = Items[Item - 1].EumItem; df.Items[i].EumUnit = Items[Item - 1].EumUnit; df.Items[i].Name = Percentiles[i].ToString() + " percentile"; } for (int i = 0; i < Percentiles.Count(); i++) { df.WriteItemTimeStep(0, i + 1, OutData[i]); } }
public void DebugPrint(string Directory) { //We need to process data for extra output while we have the particles { foreach (var c in Catchments.Where(ca => ca.EndParticles.Count >= 1)) { if (c.EndParticles.Count >= 20) { c.ParticleBreakthroughCurves = new List<Tuple<double, double>>(); MathNet.Numerics.Statistics.Percentile p = new MathNet.Numerics.Statistics.Percentile(c.EndParticles.Select(pa => pa.TravelTime)); for (int i = 1; i < np; i++) { c.ParticleBreakthroughCurves.Add(new Tuple<double, double>(i / np * 100.0, p.Compute(i / np))); } //Also do oxidized breakthrough curves if (c.EndParticles.Count(pp => pp.Registration != 1) >= 20) { c.ParticleBreakthroughCurvesOxidized = new List<Tuple<double, double>>(); p = new MathNet.Numerics.Statistics.Percentile(c.EndParticles.Where(pp => pp.Registration != 1).Select(pa => pa.TravelTime)); for (int i = 1; i < np; i++) { c.ParticleBreakthroughCurvesOxidized.Add(new Tuple<double, double>(i / np * 100.0, p.Compute(i / np))); } } } DataRow row = DebugData.Rows.Find(c.ID); if (row == null) { row = DebugData.NewRow(); row[0] = c.ID; DebugData.Rows.Add(row); } row["PartCount"] = c.EndParticles.Count; row["RedoxCount"] = c.EndParticles.Count(pp => pp.Registration == 1); row["RedoxRatio"] = c.EndParticles.Count(pp => pp.Registration == 1) / (double)c.EndParticles.Count; if (c.EndParticles.Count > 0) { row["Drain_to_River"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.Drain_to_River) / (double)c.EndParticles.Count; row["Drain_to_Boundary"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.Drain_to_Boundary) / (double)c.EndParticles.Count; row["Unsaturated_zone"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.Unsaturated_zone) / (double)c.EndParticles.Count; row["River"] = c.EndParticles.Count(pa => pa.SinkType == SinkType.River) / (double)c.EndParticles.Count; } row["PartCount_start"] = c.StartParticles.Count; } } NewMessage("Writing breakthrough curves"); var selectedCatchments = Catchments.Where(cc => cc.ParticleBreakthroughCurves != null); using (System.IO.StreamWriter sw = new System.IO.StreamWriter(Path.Combine(Directory, "BC.csv"))) { StringBuilder headline = new StringBuilder(); headline.Append("ID\tNumber of Particles"); for (int i = 1; i < np; i++) { headline.Append("\t + " + (i / np * 100.0)); } sw.WriteLine(headline); foreach (var c in selectedCatchments.Where(cc => cc.ParticleBreakthroughCurves != null)) { StringBuilder line = new StringBuilder(); line.Append(c.ID + "\t" + c.EndParticles.Count); foreach (var pe in c.ParticleBreakthroughCurves) { line.Append("\t" + pe.Item2); } sw.WriteLine(line); } } if (selectedCatchments.Count() > 0) { using (ShapeWriter sw = new ShapeWriter(Path.Combine(Directory, Name + "_debug.shp"))) { foreach (var bc in selectedCatchments.First().ParticleBreakthroughCurves) { DebugData.Columns.Add(((int)bc.Item1).ToString(), typeof(double)); } foreach (var bc in selectedCatchments.First().ParticleBreakthroughCurves) { DebugData.Columns.Add(((int)bc.Item1).ToString() + "Ox", typeof(double)); } foreach (var c in selectedCatchments) { GeoRefData gd = new GeoRefData() { Geometry = c.Geometry }; var row = DebugData.Rows.Find(c.ID); if (c.ParticleBreakthroughCurves != null) foreach (var bc in c.ParticleBreakthroughCurves) row[((int)bc.Item1).ToString()] = bc.Item2; if (c.ParticleBreakthroughCurvesOxidized != null) foreach (var bc in c.ParticleBreakthroughCurvesOxidized) row[((int)bc.Item1).ToString() + "Ox"] = bc.Item2; gd.Data = row; sw.Write(gd); } } } //selectedCatchments = Catchments.Where(cc => cc.EndParticles.Count > 0).ToList(); //foreach (var c in selectedCatchments) //{ // DataTable dt = new DataTable(); // dt.Columns.Add("Part_Id", typeof(int)); // dt.Columns.Add("Sink", typeof(string)); // dt.Columns.Add("Reg", typeof(int)); // using (ShapeWriter sw = new ShapeWriter(Path.Combine(Directory, c.ID + "_particles.shp")) { Projection = MainModel.projection }) // { // foreach (var p in c.EndParticles) // { // var row = dt.NewRow(); // row["Part_Id"] = p.ID; // row["Sink"] = p.SinkType.ToString(); // row["Reg"] = p.Registration; // sw.Write(new GeoRefData() { Geometry = new XYLine(p.XStart, p.YStart, p.X, p.Y), Data = row }); // } // } //} }