public void GhcnTempUncertaintyTest() { string etopoLocalUri = TestConstants.UriEtopo; var ghcnStorage = TestDataStorageFactory.GetStorage(TestConstants.UriGHCN); var etopoStorage = TestDataStorageFactory.GetStorage(etopoLocalUri); GHCNDataHandler handler = new GHCNDataHandler(ghcnStorage); ETOPO1DataSource.ETOPO1DataHandler elevationHandler = new ETOPO1DataSource.ETOPO1DataHandler(etopoStorage); TimeRegion tr = new TimeRegion(firstYear: 1921, lastYear: 1921).GetMonthlyTimeseries(firstMonth: 3, lastMonth: 3); //data index 2642 FetchDomain domain = FetchDomain.CreatePoints( new double[] { 36.27 }, //exact station. data index 3776 new double[] { -90.97 }, tr); FetchRequest tempRequest = new FetchRequest("temp", domain); Func <FetchRequest, Array> elevHandling = req => { var rewrittenReq = new FetchRequest("Elevation", req.Domain); return(elevationHandler.AggregateAsync(RequestContextStub.GetStub(etopoStorage, rewrittenReq), null).Result); }; var reqContext = RequestContextStub.GetStub(ghcnStorage, tempRequest, elevHandling); var compCont = new ComputationalContext(); Assert.AreEqual(0.0, (double)handler.EvaluateAsync(reqContext, compCont).Result.GetValue(0), 1e-6); //manual data comparison. }
public void GHCNplaneInterpolationCellTest() { //with lapse rate correction FetchDomain fd = FetchDomain.CreateCells(new double[] { 1.5 }, new double[] { 1.0 }, new double[] { 1.6 }, new double[] { 1.1 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april FetchRequest fr = new FetchRequest("temp", fd); GhcnPlaneStorageContext storage = new GhcnPlaneStorageContext(fr); GHCNDataHandler sotdh = new GHCNDataHandler(storage); var compCont = new ComputationalContext(); var evRes = sotdh.EvaluateAsync(storage, compCont).Result; var result = (double[])sotdh.AggregateAsync(storage, compCont).Result; Assert.AreEqual(-7.0, result[0], TestConstants.DoublePrecision); //without lapse rate correction fd = FetchDomain.CreateCells(new double[] { -2.5 }, new double[] { 1.0 }, new double[] { -2.4 }, new double[] { 1.1 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april fr = new FetchRequest("temp", fd); storage = new GhcnPlaneStorageContext(fr); sotdh = new GHCNDataHandler(storage); compCont = new ComputationalContext(); evRes = sotdh.EvaluateAsync(storage, compCont).Result; result = (double[])sotdh.AggregateAsync(storage, compCont).Result; Assert.AreEqual(-6.0, result[0], TestConstants.DoublePrecision); }
public void GHCNtempInterpolationPointsTest() { FetchDomain fd = FetchDomain.CreatePoints(new double[] { 1.5 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 60, lastDay: 90)); //mar FetchRequest fr = new FetchRequest("temp", fd); GhcnStorageContext storage = new GhcnStorageContext(fr); GHCNDataHandler sotdh = new GHCNDataHandler(storage); var compCont = new ComputationalContext(); var evRes = sotdh.EvaluateAsync(storage, compCont).Result; var result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 3/5*lat-1 = 3/5*(1.5)-1 = -0.1 Assert.AreEqual(-0.1, result[0], TestConstants.DoublePrecision); fd = FetchDomain.CreatePoints(new double[] { -2.5 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 60, lastDay: 90)); //mar fr = new FetchRequest("temp", fd); storage = new GhcnStorageContext(fr); sotdh = new GHCNDataHandler(storage); compCont = new ComputationalContext(); evRes = sotdh.EvaluateAsync(storage, compCont).Result; result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 3/5*lat = 3/5*(-2.5) = -1.5 Assert.AreEqual(-1.5, result[0], TestConstants.DoublePrecision); }
public void GHCNprateInterpolationPointsTest() { FetchDomain fd = FetchDomain.CreatePoints(new double[] { 0.5 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april FetchRequest fr = new FetchRequest("prate", fd); GhcnStorageContext storage = new GhcnStorageContext(fr); GHCNDataHandler sotdh = new GHCNDataHandler(storage); var compCont = new ComputationalContext(); var evRes = sotdh.EvaluateAsync(storage, compCont).Result; var result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 6/5*lat = 6/5*(0.5) = -3 Assert.AreEqual(0.6, result[0], TestConstants.DoublePrecision); fd = FetchDomain.CreatePoints(new double[] { -5.0 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april fr = new FetchRequest("prate", fd); storage = new GhcnStorageContext(fr); sotdh = new GHCNDataHandler(storage); compCont = new ComputationalContext(); evRes = sotdh.EvaluateAsync(storage, compCont).Result; result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 6/5*lat = 6/5*(-2.5) = -3 Assert.AreEqual(0.0, result[0], TestConstants.DoublePrecision); //as prate can't be negative, it is coerced to zero }
public void GHCNnanForOutOfObsConvexHullTest() { FetchDomain fd = FetchDomain.CreatePoints(new double[] { -10 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 60, lastDay: 90)); //mar FetchRequest fr = new FetchRequest("prate", fd); GhcnStorageContext storage = new GhcnStorageContext(fr); GHCNDataHandler sotdh = new GHCNDataHandler(storage); var compCont = new ComputationalContext(); var evRes = (double[])sotdh.EvaluateAsync(storage, compCont).Result; Assert.IsTrue(double.IsNaN(evRes[0])); var result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //must be NAN as requested point out of observations convex hull Assert.IsTrue(double.IsNaN(result[0])); }
public void Bug1691() { double BayOfBiscaySELat = 44.5; double BayOfBiscaySELon = -3.5; double InFranceLat = 47; double InFranceLon = 1; Random r = new Random(1); var eps = r.NextDouble() / 10.0; double latDelta = InFranceLat - BayOfBiscaySELat; double lonDelta = InFranceLon - BayOfBiscaySELon; var tr = new TimeRegion(1990, 2001, 1, -1, 0, 24, true, false, true); var request = new FetchRequest( "temp", FetchDomain.CreateCellGrid( Enumerable.Range(0, 31).Select(i => eps + BayOfBiscaySELat + i * latDelta / 31.0).ToArray(), Enumerable.Range(0, 21).Select(i => eps + BayOfBiscaySELon + i * lonDelta / 21.0).ToArray(), tr)); string etopoLocalUri = TestConstants.UriEtopo; var ghcnStorage = TestDataStorageFactory.GetStorage(TestConstants.UriGHCN); var etopoStorage = TestDataStorageFactory.GetStorage(etopoLocalUri); GHCNDataHandler handler = new GHCNDataHandler(ghcnStorage); ETOPO1DataSource.ETOPO1DataHandler elevationHandler = new ETOPO1DataSource.ETOPO1DataHandler(etopoStorage); Func <FetchRequest, Array> elevHandling = req => { var rewrittenReq = new FetchRequest("Elevation", req.Domain); return(elevationHandler.AggregateAsync(RequestContextStub.GetStub(etopoStorage, rewrittenReq), null).Result); }; var reqContext = RequestContextStub.GetStub(ghcnStorage, request, elevHandling); var compCont = new ComputationalContext(); var evRes = handler.EvaluateAsync(reqContext, compCont).Result; Assert.IsTrue(-70.0 < (double)handler.AggregateAsync(reqContext, compCont).Result.GetValue(16, 2, 0)); //manual data comparison. }
public void GhcnPrateUncertaintyTest() { System.Diagnostics.Trace.WriteLine(TestConstants.UriGHCN); var storage = TestDataStorageFactory.GetStorage(TestConstants.UriGHCN); GHCNDataHandler handler = new GHCNDataHandler(storage); TimeRegion tr = new TimeRegion(firstYear: 1914, lastYear: 1914).GetMonthlyTimeseries(firstMonth: 12, lastMonth: 12); //data index 2567 FetchDomain domain = FetchDomain.CreatePoints( new double[] { -25.18 }, //exact station. data index 18398 new double[] { 151.65 }, tr); FetchRequest prateRequest = new FetchRequest("prate", domain); var compCont = new ComputationalContext(); var reqContext = RequestContextStub.GetStub(storage, prateRequest); Assert.AreEqual(0.0, (double)handler.EvaluateAsync(reqContext, compCont).Result.GetValue(0), 1e-6); //manual data comparison. }
public void PlaneInterpolationTest() { double[] lats = new double[] { 0.0, 5.0, 5.0 }; double[] lons = new double[] { 0.0, 0.0, 6.0 }; double[] vals = new double[] { 0.0, 3.0, 3.0 }; FetchDomain fd = FetchDomain.CreatePoints(new double[] { -5.0 }, new double[] { 6.0 }, new TimeRegion()); FetchRequest fr = new FetchRequest("val", fd); var storage = new SimplePlaneStorageContext(lats, lons, vals, fr); storage.Request = fr; SpatialOnlyTpsDataHandler sotdh = new SpatialOnlyTpsDataHandler(storage); var compContext = new ComputationalContext(); sotdh.EvaluateAsync(storage, compContext); var result = (double[])sotdh.AggregateAsync(storage, compContext).Result; Assert.AreEqual(-3.0, result[0], TestConstants.DoublePrecision); }
public void SparsePointsYieldNansOrPlane() { double[] lats = new double[] { 0.0, 5.0, 50.0 }; double[] lons = new double[] { 0.0, 0.0, 0.0 }; double[] vals = new double[] { 1.0, 1.0, 1.0 }; FetchDomain fd = FetchDomain.CreatePoints(new double[] { 7.0 }, new double[] { 6.0 }, new TimeRegion()); FetchRequest fr = new FetchRequest("val", fd); var storage = new SimplePlaneStorageContext(lats, lons, vals, fr); storage.Request = fr; SpatialOnlyTpsDataHandler sotdh = new SpatialOnlyTpsDataHandler(storage); ComputationalContext cc = new ComputationalContext(); var unc = sotdh.EvaluateAsync(storage, cc).Result; var result = (double[])sotdh.AggregateAsync(storage, cc).Result; Assert.IsTrue(double.IsNaN(result[0]) || Math.Abs(1.0 - result[0]) < TestConstants.FloatPrecision); }
protected virtual async Task <double> CalculateCellMean(IRequestContext requestContext, ComputationalContext computationalContext, double latmin, double latmax, double lonmin, double lonmax, double[] obsLats, double[] obsLons, double[] obsVals) { if (latmin == latmax && lonmin == lonmax) { return(spatialIntegrator.Interpolate(latmin, lonmin, obsLats, obsLons, obsVals)); } //non-zero area var interpolationContext = spatialIntegrator.GetInterpolationContext(obsLats, obsLons, obsVals); double latStep = (latmax - latmin) / 19.0; double lonStep = (lonmax - lonmin) / 19.0; double acc = 0.0; double r; for (int i = 0; i < 20; i++) { for (int j = 0; j < 20; j++) { r = spatialIntegrator.Interpolate(latmin + latStep * i, lonmin + lonStep * j, interpolationContext); acc += r; } } return(acc / 400.0); }
/// <summary> /// Returns a set of linear weights along with timeSegmint for with dataIndeces in linear weights are valid /// </summary> /// <param name="context"></param> /// <param name="computationalContext"></param> /// <param name="cells"></param> /// <returns></returns> protected IEnumerable <Tuple <ITimeSegment, LinearWeight[]> > CalcLinearWeights(ComputationalContext computationalContext, IEnumerable <GeoCellTuple> cells) { if (!computationalContext.ContainsKey("observations")) { throw new InvalidOperationException("Call SaveObservationsAndDalanayDiagsForCells prior calling CompleteAggregateCellsBatch"); } ISpatPointsLinearInterpolator2D spli2d = (ISpatPointsLinearInterpolator2D)spatialIntegrator; Dictionary <ITimeSegment, IObservationsInformation> observations = (Dictionary <ITimeSegment, IObservationsInformation>)computationalContext["observations"]; Dictionary <ITimeSegment, object> delanays = (Dictionary <ITimeSegment, object>)computationalContext["dalanays"]; var sw1 = System.Diagnostics.Stopwatch.StartNew(); TraceVerbose("Computing field values"); //WARNING: can't be paralleled as dalanays are not thread safe foreach (var cell in cells) { var timeSegment = cell.Time; var observation = observations[timeSegment].Observations; object delanay = delanays[timeSegment]; double latmax = cell.LatMax, latmin = cell.LatMin, lonmax = cell.LonMax, lonmin = cell.LonMin; if (latmax == latmin && lonmax == lonmin) { yield return(Tuple.Create(timeSegment, spli2d.GetLinearWeigths(latmin, lonmin, delanay))); } else { var sizeSqrt = (int)Math.Sqrt(cellAveragingGridSize); LinearWeight[][] toFlatten = new LinearWeight[sizeSqrt * sizeSqrt][]; double latStep = (latmax - latmin) / (sizeSqrt - 1); double lonStep = (lonmax - lonmin) / (sizeSqrt - 1); for (int i = 0; i < sizeSqrt; i++) { for (int j = 0; j < sizeSqrt; j++) { var w = spli2d.GetLinearWeigths(latmin + latStep * i, lonmin + lonStep * j, delanay); toFlatten[sizeSqrt * i + j] = w; } } var devisor = sizeSqrt * sizeSqrt; var flattenedWeights = toFlatten.SelectMany(weights => weights).GroupBy(w => w.DataIndex).Select(g => new LinearWeight(g.Key, g.Select(g1 => g1.Weight).Sum() / devisor)).ToArray(); yield return(Tuple.Create(timeSegment, flattenedWeights)); } } TraceVerbose("Computing field values finished in {0}", sw1.Elapsed); }
/// <summary> /// Computes delanay triangulation for each unique time segment that is present in the cells enumeration. Saves them into comp context /// </summary> /// <param name="context"></param> /// <param name="computationalContext"></param> /// <param name="cells"></param> /// <returns></returns> protected async Task SaveObservationsAndDalanayDiagsForCells(IRequestContext context, ComputationalContext computationalContext, IEnumerable <GeoCellTuple> cells) { var variable = context.Request.EnvironmentVariableName; var cellsArray = cells.ToArray(); var numberedCells = Enumerable.Zip(cellsArray, Enumerable.Range(0, int.MaxValue), (cell, num) => Tuple.Create(cell, num)).ToArray(); var missingValue = MissingValuesDictionary[variable]; int count = numberedCells.Length; Dictionary <ITimeSegment, IObservationsInformation> observations = new Dictionary <ITimeSegment, IObservationsInformation>(); Dictionary <ITimeSegment, object> delanays = new Dictionary <ITimeSegment, object>(); var timeGroups = numberedCells.GroupBy(cell => cell.Item1.Time).ToArray(); //fetching observations for different timeSegments, constructing delanay triangulations for them. storing them into the comp context TraceVerbose("Fetching observations for different time segments, building delanays in parallel"); var timeSegmentGroupedTasks = timeGroups.Select(group => Task.Run(async() => { var timeSegment = group.Key; int groupHashCode = group.GetHashCode(); TraceVerbose("Getting observations for time segment {0}", groupHashCode); System.Diagnostics.Stopwatch sw1 = System.Diagnostics.Stopwatch.StartNew(); var observation = await observationProvider.GetObservationsAsync(context, variable, missingValue, 0.0, 0.0, 0.0, 0.0, timeSegment); sw1.Stop(); TraceVerbose("Got observations for time segment {0} in {1}", groupHashCode, sw1.Elapsed); double[] lats = observation.Observations.Select(o => o.Latitude).ToArray(); double[] lons = observation.Observations.Select(o => o.Longitude).ToArray(); double[] vals = observation.Observations.Select(o => o.Value).ToArray(); TraceVerbose("Generating dalanay for time segment {0} ({1} observations)", groupHashCode, observation.Observations.Length); System.Diagnostics.Stopwatch sw2 = System.Diagnostics.Stopwatch.StartNew(); var dalanay = nni.GetInterpolationContext(lats, lons, vals); sw2.Stop(); TraceVerbose("Generated dalanay for time segment {0} ({1} observations) in {2}", groupHashCode, observation.Observations.Length, sw2.Elapsed); observations.Add(timeSegment, observation); delanays.Add(timeSegment, dalanay); })); var syncTask = Task.WhenAll(timeSegmentGroupedTasks); await syncTask; TraceVerbose("Fetched observations for different time segments. Delanay triangulations are computed"); computationalContext["observations"] = observations; computationalContext["dalanays"] = delanays; computationalContext["requestCells"] = cellsArray; }
protected async override Task <double[]> AggregateCellsBatchAsync(IRequestContext context, ComputationalContext computationalContext, IEnumerable <GeoCellTuple> cells) { await SaveObservationsAndDalanayDiagsForCells(context, computationalContext, cells); Dictionary <ITimeSegment, IObservationsInformation> observations = (Dictionary <ITimeSegment, IObservationsInformation>)computationalContext["observations"]; var res = CalcLinearWeights(computationalContext, cells).Select(t => { var releventObservations = observations[t.Item1].Observations; return(t.Item2.Sum(x => x.Weight * releventObservations[x.DataIndex].Value)); }).ToArray(); return(res); }
protected override async Task <double[]> EvaluateCellsBatchAsync(IRequestContext context, ComputationalContext computationalContext, IEnumerable <GeoCellTuple> cells) { await SaveObservationsAndDalanayDiagsForCells(context, computationalContext, cells); VariogramModule.IVariogramFitter variogramFitter = new LMDotNetVariogramFitter.Fitter(); Dictionary <ITimeSegment, VariogramModule.IVariogram> variograms = new Dictionary <ITimeSegment, VariogramModule.IVariogram>(); Dictionary <ITimeSegment, IObservationsInformation> observations = (Dictionary <ITimeSegment, IObservationsInformation>)computationalContext["observations"]; LimitedConcurrencyLevelTaskScheduler lclts = new LimitedConcurrencyLevelTaskScheduler(Environment.ProcessorCount); TaskFactory taskFactory = new TaskFactory(lclts); var variogramTasks = observations.Select(pair => taskFactory.StartNew(() => { ITimeSegment ts = pair.Key; TraceVerbose(string.Format("Fitting variogram for {0} ({1} observations)", ts, pair.Value.Observations.Length)); Stopwatch sw1 = Stopwatch.StartNew(); var lats = pair.Value.Observations.Select(o => o.Latitude).ToArray(); var lons = pair.Value.Observations.Select(o => o.Longitude).ToArray(); var vals = pair.Value.Observations.Select(o => o.Value).ToArray(); var pointSet = new EmpVariogramBuilder.PointSet(lats, lons, vals); var dist = FuncConvert.ToFSharpFunc(new Converter <Tuple <double, double>, FSharpFunc <Tuple <double, double>, double> >(t1 => FuncConvert.ToFSharpFunc(new Converter <Tuple <double, double>, double>(t2 => SphereMath.GetDistance(t1.Item1, t1.Item2, t2.Item1, t2.Item2))))); var empVar = EmpVariogramBuilder.EmpiricalVariogramBuilder.BuildEmpiricalVariogram(pointSet, dist); var fitted_variogram = variogramFitter.Fit(empVar); VariogramModule.IVariogram effectiveVariogram = null; sw1.Stop(); if (FSharpOption <VariogramModule.IDescribedVariogram> .get_IsSome(fitted_variogram)) { effectiveVariogram = fitted_variogram.Value; TraceVerbose(string.Format("Variogram fited for {0} ({1} observations) in {2}", ts, pair.Value.Observations.Length, sw1.Elapsed)); } else { TraceWarning(string.Format("Variogram fitting failed for {0} ({1} observations) in {2}. Using fallback variogram", ts, pair.Value.Observations.Length, sw1.Elapsed)); effectiveVariogram = variogramFitter.GetFallback(empVar); } lock ("saving_variograms") { variograms.Add(ts, effectiveVariogram); } })); TraceVerbose(string.Format("Starting calculations of linear weights for all cells")); Stopwatch sw2 = Stopwatch.StartNew(); var weigths = CalcLinearWeights(computationalContext, cells); sw2.Stop(); TraceVerbose(string.Format("calculations of linear weights for all cells ended in {0}", sw2.Elapsed)); TraceVerbose(string.Format("Waiting for all variograms to be computed")); await Task.WhenAll(variogramTasks); TraceVerbose(string.Format("All variograms are computed. Calculating variances and values")); Stopwatch sw3 = Stopwatch.StartNew(); var resultValues = cells.Zip(weigths, (cell, weightTuple) => { ITimeSegment ts = cell.Time; var weight = weightTuple.Item2; VariogramModule.IVariogram variogram = variograms[ts]; var observation = observations[ts].Observations; Debug.Assert(Math.Abs(weight.Sum(w => w.Weight) - 1.0) < 1e-10); double sill = variogram.Sill; double cellLat = (cell.LatMax + cell.LatMin) * 0.5; double cellLon = (cell.LonMax + cell.LonMin) * 0.5; //var = cov(0)+ sum sum (w[i]*w[j]*cov(i,j))-2.0*sum(w[i]*cov(x,i)) double cov_at_0 = sill; double acc = cov_at_0; //cov(0) for (int i = 0; i < weight.Length; i++) { double w = weight[i].Weight; int idx1 = weight[i].DataIndex; double lat1 = observation[idx1].Latitude; double lon1 = observation[idx1].Longitude; for (int j = 0; j < i; j++) { int idx2 = weight[j].DataIndex; double lat2 = observation[idx2].Latitude; double lon2 = observation[idx2].Longitude; double dist = SphereMath.GetDistance(lat1, lon1, lat2, lon2); double cov = sill - variogram.GetGamma(dist); acc += 2.0 * w * weight[j].Weight * cov; } acc += w * w * cov_at_0; //diagonal elements double dist2 = SphereMath.GetDistance(lat1, lon1, cellLat, cellLon); double cov2 = sill - variogram.GetGamma(dist2); acc -= 2.0 * w * cov2; } return(Tuple.Create(cell, Math.Sqrt(acc), weight.Sum(w => observation[w.DataIndex].Value * w.Weight))); }).ToArray(); sw3.Stop(); TraceVerbose(string.Format("All sigmas calulated in {0}", sw3.Elapsed)); computationalContext.Add("results", resultValues); return(resultValues.Select(r => r.Item2).ToArray()); }
protected async override Task <double[]> AggregateCellsBatchAsync(IRequestContext context, ComputationalContext computationalContext, IEnumerable <GeoCellTuple> cells) { TraceVerbose(string.Format("Aggregation: exracting already computed values")); Stopwatch sw = Stopwatch.StartNew(); Tuple <GeoCellTuple, double, double>[] precomputed = (Tuple <GeoCellTuple, double, double>[])computationalContext["results"]; int idx = 0; GeoCellTuple[] cellsArray = cells.ToArray(); int n = cellsArray.Length; double[] res = new double[n]; for (int aggIdx = 0; aggIdx < n; aggIdx++) { while (!cellsArray[aggIdx].Equals(precomputed[idx].Item1)) { idx++; } res[aggIdx] = precomputed[idx].Item3; } sw.Stop(); TraceVerbose(string.Format("Aggregation: extracted values in {0}", sw.Elapsed)); return(res); }