public void GHCNplaneInterpolationCellTest()
{
//with lapse rate correction
FetchDomain fd = FetchDomain.CreateCells(new double[] { 1.5 }, new double[] { 1.0 }, new double[] { 1.6 }, new double[] { 1.1 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april
FetchRequest fr = new FetchRequest("temp", fd);
GhcnPlaneStorageContext storage = new GhcnPlaneStorageContext(fr);
GHCNDataHandler sotdh = new GHCNDataHandler(storage);
var compCont = new ComputationalContext();
var evRes = sotdh.EvaluateAsync(storage, compCont).Result;
var result = (double[])sotdh.AggregateAsync(storage, compCont).Result;
Assert.AreEqual(-7.0, result[0], TestConstants.DoublePrecision);
//without lapse rate correction
fd = FetchDomain.CreateCells(new double[] { -2.5 }, new double[] { 1.0 }, new double[] { -2.4 }, new double[] { 1.1 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april
fr = new FetchRequest("temp", fd);
storage = new GhcnPlaneStorageContext(fr);
sotdh = new GHCNDataHandler(storage);
compCont = new ComputationalContext();
evRes = sotdh.EvaluateAsync(storage, compCont).Result;
result = (double[])sotdh.AggregateAsync(storage, compCont).Result;
Assert.AreEqual(-6.0, result[0], TestConstants.DoublePrecision);
}
public void GhcnTempUncertaintyTest()
{
string etopoLocalUri = TestConstants.UriEtopo;
var ghcnStorage = TestDataStorageFactory.GetStorage(TestConstants.UriGHCN);
var etopoStorage = TestDataStorageFactory.GetStorage(etopoLocalUri);
GHCNDataHandler handler = new GHCNDataHandler(ghcnStorage);
ETOPO1DataSource.ETOPO1DataHandler elevationHandler = new ETOPO1DataSource.ETOPO1DataHandler(etopoStorage);
TimeRegion tr = new TimeRegion(firstYear: 1921, lastYear: 1921).GetMonthlyTimeseries(firstMonth: 3, lastMonth: 3); //data index 2642
FetchDomain domain = FetchDomain.CreatePoints(
new double[] { 36.27 }, //exact station. data index 3776
new double[] { -90.97 },
tr);
FetchRequest tempRequest = new FetchRequest("temp", domain);
Func <FetchRequest, Array> elevHandling = req =>
{
var rewrittenReq = new FetchRequest("Elevation", req.Domain);
return(elevationHandler.AggregateAsync(RequestContextStub.GetStub(etopoStorage, rewrittenReq), null).Result);
};
var reqContext = RequestContextStub.GetStub(ghcnStorage, tempRequest, elevHandling);
var compCont = new ComputationalContext();
Assert.AreEqual(0.0, (double)handler.EvaluateAsync(reqContext, compCont).Result.GetValue(0), 1e-6); //manual data comparison.
}
public void GHCNtempInterpolationPointsTest()
{
FetchDomain fd = FetchDomain.CreatePoints(new double[] { 1.5 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 60, lastDay: 90)); //mar
FetchRequest fr = new FetchRequest("temp", fd);
GhcnStorageContext storage = new GhcnStorageContext(fr);
GHCNDataHandler sotdh = new GHCNDataHandler(storage);
var compCont = new ComputationalContext();
var evRes = sotdh.EvaluateAsync(storage, compCont).Result;
var result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 3/5*lat-1 = 3/5*(1.5)-1 = -0.1
Assert.AreEqual(-0.1, result[0], TestConstants.DoublePrecision);
fd = FetchDomain.CreatePoints(new double[] { -2.5 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 60, lastDay: 90)); //mar
fr = new FetchRequest("temp", fd);
storage = new GhcnStorageContext(fr);
sotdh = new GHCNDataHandler(storage);
compCont = new ComputationalContext();
evRes = sotdh.EvaluateAsync(storage, compCont).Result;
result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 3/5*lat = 3/5*(-2.5) = -1.5
Assert.AreEqual(-1.5, result[0], TestConstants.DoublePrecision);
}
public void MarksRequestTest()
{
double[] latmin = null, latmax, lonmin, lonmax = null;
int[] startday, stopday, startyear, stopyear, starthour, stophour;
FetchRequest fr = null;
using (DataSet ds = DataSet.Open(@"423b28bf6a4357f14b64f2b16ab759cb6b5961db.csv?openMode=readOnly"))
{
latmin = ((double[])ds.Variables["LatMin"].GetData()).Select(e => (double)e).ToArray();
latmax = ((double[])ds.Variables["LatMax"].GetData()).Select(e => (double)e).ToArray();
lonmin = ((double[])ds.Variables["LonMin"].GetData()).Select(e => (double)e).ToArray();
lonmax = ((double[])ds.Variables["LonMax"].GetData()).Select(e => (double)e).ToArray();
startday = ((int[])ds.Variables["StartDay"].GetData());
stopday = ((int[])ds.Variables["StopDay"].GetData());
starthour = ((int[])ds.Variables["StartHour"].GetData());
stophour = ((int[])ds.Variables["StartHour"].GetData());
startyear = ((int[])ds.Variables["StartYear"].GetData());
stopyear = ((int[])ds.Variables["StartYear"].GetData());
TimeRegion tr = new TimeRegion(startyear[0], stopyear[0]);
fr = new FetchRequest("prate", FetchDomain.CreatePoints(latmin, lonmax, tr), new string[] { "CRU CL 2.0" });
}
var request = JobManager.EvaluateSubrequestData(fr, 18, 16);
}
public FetchRequest(FetchClimate2.IFetchRequest request)
{
EnvironmentVariableName = request.EnvironmentVariableName;
ReproducibilityTimestamp = DateTime.SpecifyKind(request.ReproducibilityTimestamp, DateTimeKind.Utc);
ParticularDataSources = request.ParticularDataSource;
Domain = new FetchDomain(request.Domain);
}
public async Task Bug1691()
{
double BayOfBiscaySELat = 44.5;
double BayOfBiscaySELon = -3.5;
double InFranceLat = 47;
double InFranceLon = 1;
Random r = new Random(1);
var eps = r.NextDouble() / 10.0;
double latDelta = InFranceLat - BayOfBiscaySELat;
double lonDelta = InFranceLon - BayOfBiscaySELon;
var tr = new TimeRegion(1990, 2001, 1, -1, 0, 24);
var request = new FetchRequest(
"temp",
FetchDomain.CreateCellGrid(
Enumerable.Range(0, 31).Select(i => eps + BayOfBiscaySELat + i * latDelta / 31.0).ToArray(),
Enumerable.Range(0, 21).Select(i => eps + BayOfBiscaySELon + i * lonDelta / 21.0).ToArray(),
tr));
string etopoLocalUri = TestConstants.UriEtopo;
var ghcnStorage = TestDataStorageFactory.GetStorageContext(TestConstants.UriGHCN);
var reqContext = RequestContextStub.GetStub(ghcnStorage, request);
DataSourceHandler handler = await GHCNv2DataSource.DataHandler.CreateAsync(ghcnStorage);
var result = await handler.ProcessRequestAsync(reqContext);
Assert.IsTrue(-70.0 < (double)result.GetValue(16, 2)); //manual data comparison.
}
public async Task TestLinearTransformDecorator()
{
var component = new ValAggStub();
var storage = TestDataStorageFactory.GetStorage("msds:memory");
storage.StorageDefinition.VariablesMetadata.Add("a", new MetaDataDictionary());
storage.StorageDefinition.VariablesMetadata["a"]["scale_factor"] = 13.0;
storage.StorageDefinition.VariablesMetadata["a"]["add_offset"] = 17.0;
var dec = new Microsoft.Research.Science.FetchClimate2.ValueAggregators.LinearTransformDecorator(storage, component);
dec.SetAdditionalTranform("a", b => b * 3 + 7.0);
FetchRequest fr = new FetchRequest("a", FetchDomain.CreatePoints(new double[] { 5.0 }, new double[] { -11.0 }, new TimeRegion()));
IRequestContext rcs = RequestContextStub.GetStub(storage, fr);
var res = await dec.AggregateCellsBatchAsync(rcs, new GeoCellTuple[] { new GeoCellTuple()
{
LatMax = 5.0, LatMin = 5.0, LonMax = -11.0, LonMin = -11.0, Time = new TimeSegment()
} });
Assert.AreEqual(253.0, res[0]);
}
public async Task CruAllVaraiblesValuesTest()
{
var storage = TestDataStorageFactory.GetStorageContext(TestConstants.UriCru);
CruCl20DataHandler cru = await CruCl20DataHandler.CreateAsync(storage);
ITimeRegion tr = new TimeRegion().GetMonthlyTimeseries(firstMonth: 1, lastMonth: 1);
IFetchDomain domain = FetchDomain.CreatePoints(
new double[] { 48.25 }, // data index 679
new double[] { -100.25 }, //data index 478
tr);
var handlerPrivate = new PrivateObject(cru, new PrivateType(typeof(DataHandlerFacade)));
var aggregatorPrivate = new PrivateObject(handlerPrivate, "valuesAggregator");
FetchRequest tmpRequest = new FetchRequest("tmp", domain);
FetchRequest preRequest = new FetchRequest("pre", domain);
FetchRequest wndRequest = new FetchRequest("wnd", domain);
FetchRequest sunpRequest = new FetchRequest("sunp", domain);
FetchRequest rehRequest = new FetchRequest("reh", domain);
FetchRequest rd0Request = new FetchRequest("rd0", domain);
FetchRequest frsRequest = new FetchRequest("frs", domain);
FetchRequest dtrRequest = new FetchRequest("dtr", domain);
Assert.AreEqual(4.9, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, wndRequest), null))).GetValue(0), TestConstants.FloatPrecision); //manual data comparision
Assert.AreEqual(-15.4, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, tmpRequest), null))).GetValue(0), TestConstants.FloatPrecision);
Assert.AreEqual(48.1, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, sunpRequest), null))).GetValue(0), TestConstants.FloatPrecision);
Assert.AreEqual(74.5, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, rehRequest), null))).GetValue(0), TestConstants.FloatPrecision);
Assert.AreEqual(7.6, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, rd0Request), null))).GetValue(0), TestConstants.FloatPrecision);
Assert.AreEqual(14.0, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, preRequest), null))).GetValue(0), TestConstants.FloatPrecision);
Assert.AreEqual(30.7, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, frsRequest), null))).GetValue(0), TestConstants.FloatPrecision);
Assert.AreEqual(11.8, (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, dtrRequest), null))).GetValue(0), TestConstants.FloatPrecision);
}
public async Task TwoFederatedDsOnSingleRemoteServiceTest()
{
FederationTestConfigurationProvider provider = new FederationTestConfigurationProvider();
Dictionary <string, StaticValuesForRectDataHandler[]> federatedDsDict = new Dictionary <string, StaticValuesForRectDataHandler[]>();
federatedDsDict["http://1/"] = new StaticValuesForRectDataHandler[] { new Static_25_45_Handler(), new Static_30_50_Handler() };
federatedDsDict["http://2/"] = new StaticValuesForRectDataHandler[] { new Static_35_55_Handler() };
FederatedDsTestingFetchEngine fetchEngine = new FederatedDsTestingFetchEngine(provider, federatedDsDict);
FetchDomain fd = FetchDomain.CreatePoints(new double[] { 5.0, 5.0, 5.0, 5.0 }, new double[] { 37.0, 42.0, 47.0, 52.0 }, new TimeRegion());
FetchRequest request = new FetchRequest("a", fd, new string[] { "25_45", "30_50" });
FetchResponseWithProvenance result = await fetchEngine.PerformRequestAsync(request);
Assert.AreEqual(-60.0, result.Values.GetValue(0));
Assert.AreEqual(60.0, result.Uncertainty.GetValue(0));
Assert.AreEqual((ushort)1, result.Provenance.GetValue(0));
Assert.AreEqual(-60.0, result.Values.GetValue(1));
Assert.AreEqual(60.0, result.Uncertainty.GetValue(1));
Assert.AreEqual((ushort)1, result.Provenance.GetValue(1));
Assert.AreEqual(-70.0, result.Values.GetValue(2));
Assert.AreEqual(70.0, result.Uncertainty.GetValue(2));
Assert.AreEqual((ushort)2, result.Provenance.GetValue(2));
Assert.AreEqual(-160.0, result.Values.GetValue(3));
Assert.AreEqual(160.0, result.Uncertainty.GetValue(3));
Assert.AreEqual((ushort)1, result.Provenance.GetValue(3));
}
public async Task FederatedFetchingNanHandlingTest()
{
FederationTestConfigurationProvider provider = new FederationTestConfigurationProvider();
Dictionary <string, StaticValuesForRectDataHandler[]> federatedDsDict = new Dictionary <string, StaticValuesForRectDataHandler[]>();
federatedDsDict["http://1/"] = new StaticValuesForRectDataHandler[] { new Static_25_45_Handler(), new StaticNanYielding_30_50_Handler() };
federatedDsDict["http://2/"] = new StaticValuesForRectDataHandler[] { new Static_35_55_Handler() };
FederatedDsTestingFetchEngine fetchEngine = new FederatedDsTestingFetchEngine(provider, federatedDsDict);
FetchDomain fd = FetchDomain.CreatePoints(new double[] { 5.0, 5.0, 5.0, 5.0 }, new double[] { 37.0, 42.0, 47.0, 52.0 }, new TimeRegion());
FetchRequest request = new FetchRequest("a", fd);
FetchResponseWithProvenance result = await fetchEngine.PerformRequestAsync(request);
Assert.AreEqual(-50.0, result.Values.GetValue(0));
Assert.AreEqual(50.0, result.Uncertainty.GetValue(0));
Assert.AreEqual((ushort)0, result.Provenance.GetValue(0));
Assert.AreEqual(-60.0, result.Values.GetValue(1)); //serve
Assert.AreEqual(60.0, result.Uncertainty.GetValue(1));
Assert.AreEqual((ushort)1, result.Provenance.GetValue(1));
Assert.AreEqual(-80.0, result.Values.GetValue(2)); //here instead of StaticNanYielding_30_50_Handler, Static_35_55_Handler must provide results
Assert.AreEqual(80.0, result.Uncertainty.GetValue(2));
Assert.AreEqual((ushort)3, result.Provenance.GetValue(2));
Assert.AreEqual(-80.0, result.Values.GetValue(3));
Assert.AreEqual(80.0, result.Uncertainty.GetValue(3));
Assert.AreEqual((ushort)3, result.Provenance.GetValue(3));
}
public async Task FetchingSingleDataSourceTest()
{
FederationTestConfigurationProvider provider = new FederationTestConfigurationProvider();
Dictionary <string, StaticValuesForRectDataHandler[]> federatedDsDict = new Dictionary <string, StaticValuesForRectDataHandler[]>();
federatedDsDict["http://1/"] = new StaticValuesForRectDataHandler[] { new Static_25_45_Handler(), new Static_30_50_Handler() };
federatedDsDict["http://2/"] = new StaticValuesForRectDataHandler[] { new Static_35_55_Handler() };
FederatedDsTestingFetchEngine fetchEngine = new FederatedDsTestingFetchEngine(provider, federatedDsDict);
FetchDomain fd = FetchDomain.CreatePoints(new double[] { 5.0 }, new double[] { 37.0 }, new TimeRegion());
FetchRequest request = new FetchRequest("a", fd, new string[] { "20_40" }); //local
FetchResponseWithProvenance result = await fetchEngine.PerformRequestAsync(request);
Assert.AreEqual(-50.0, result.Values.GetValue(0));
Assert.AreEqual(50.0, result.Uncertainty.GetValue(0));
Assert.AreEqual((ushort)0, result.Provenance.GetValue(0));
request = new FetchRequest("a", fd, new string[] { "35_55" }); //federated
result = await fetchEngine.PerformRequestAsync(request);
Assert.AreEqual(-80.0, result.Values.GetValue(0));
Assert.AreEqual(80.0, result.Uncertainty.GetValue(0));
Assert.AreEqual((ushort)3, result.Provenance.GetValue(0));
}
public async Task ReanslysisUncertatintyTest()
{
var storage = TestDataStorageFactory.GetStorageContext(TestConstants.UriReanalysisRegular);
NCEPReanalysisRegularGridDataHandler regularHandler = await NCEPReanalysisRegularGridDataHandler.CreateAsync(storage);
ITimeRegion tr = new TimeRegion(firstYear: 1961, lastYear: 1990);
IFetchDomain dataDomain1 = FetchDomain.CreatePoints(
new double[] { 37.5 },
new double[] { 30.0 }, tr);
IFetchDomain dataDomain2 = FetchDomain.CreatePoints(
new double[] { 37.5 },
new double[] { -30.0 }, tr);
FetchRequest dataRequest1 = new FetchRequest("air", dataDomain1);
FetchRequest dataRequest2 = new FetchRequest("air", dataDomain2);
var handlerPrivate = new PrivateObject(regularHandler, new PrivateType(typeof(DataHandlerFacade)));
var evaluatorPrivate = new PrivateObject(handlerPrivate, "uncertaintyEvaluator");
Array dataUncertainty1 = await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, dataRequest1)));
Array dataUncertainty2 = await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, dataRequest2)));
Assert.IsFalse(double.IsNaN((double)dataUncertainty1.GetValue(0)));
Assert.IsFalse(double.IsNaN((double)dataUncertainty2.GetValue(0)));
Assert.IsTrue(((double)dataUncertainty1.GetValue(0)) < double.MaxValue);
Assert.IsTrue(((double)dataUncertainty2.GetValue(0)) < double.MaxValue);
Assert.IsTrue(((double)dataUncertainty1.GetValue(0)) >= 0.0);
Assert.IsTrue(((double)dataUncertainty2.GetValue(0)) >= 0.0);
}
public async Task ReanslysisNewYearCrossingSubDayTest()
{
var regularStorage = TestDataStorageFactory.GetStorageContext(TestConstants.UriReanalysisRegular);
NCEPReanalysisRegularGridDataHandler regularHandler = await NCEPReanalysisRegularGridDataHandler.CreateAsync(regularStorage);
ITimeRegion crossingTr = new TimeRegion(firstYear: 1950, lastYear: 1951, firstDay: 336, lastDay: 30, startHour: 0, stopHour: 12);
IFetchDomain crossingDomain = FetchDomain.CreatePoints(
new double[] { -37.5 }, // data index 51
new double[] { 137.5 }, //data index 55
crossingTr);
ITimeRegion firstTr = new TimeRegion(firstYear: 1950, lastYear: 1950, firstDay: 336, lastDay: 365, startHour: 0, stopHour: 12);
IFetchDomain firstDomain = FetchDomain.CreatePoints(
new double[] { -37.5 }, // data index 51
new double[] { 137.5 }, //data index 55
firstTr);
ITimeRegion secondTr = new TimeRegion(firstYear: 1951, lastYear: 1951, firstDay: 1, lastDay: 30, startHour: 0, stopHour: 12);
IFetchDomain secondDomain = FetchDomain.CreatePoints(
new double[] { -37.5 }, // data index 51
new double[] { 137.5 }, //data index 55
secondTr);
var handlerPrivate = new PrivateObject(regularHandler, new PrivateType(typeof(DataHandlerFacade)));
var aggregatorPrivate = new PrivateObject(handlerPrivate, "valuesAggregator");
double crossing = (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(regularStorage, new FetchRequest("air", crossingDomain)), null))).GetValue(0);
double first = (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(regularStorage, new FetchRequest("air", firstDomain)), null))).GetValue(0);
double second = (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(regularStorage, new FetchRequest("air", secondDomain)), null))).GetValue(0);
Assert.AreEqual(crossing, (first + second) / 2.0, 1e-6);
}
public void GHCNprateInterpolationPointsTest()
{
FetchDomain fd = FetchDomain.CreatePoints(new double[] { 0.5 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april
FetchRequest fr = new FetchRequest("prate", fd);
GhcnStorageContext storage = new GhcnStorageContext(fr);
GHCNDataHandler sotdh = new GHCNDataHandler(storage);
var compCont = new ComputationalContext();
var evRes = sotdh.EvaluateAsync(storage, compCont).Result;
var result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 6/5*lat = 6/5*(0.5) = -3
Assert.AreEqual(0.6, result[0], TestConstants.DoublePrecision);
fd = FetchDomain.CreatePoints(new double[] { -5.0 }, new double[] { 1.0 }, new TimeRegion(firstYear: 1970, lastYear: 1970, firstDay: 91, lastDay: 120)); //april
fr = new FetchRequest("prate", fd);
storage = new GhcnStorageContext(fr);
sotdh = new GHCNDataHandler(storage);
compCont = new ComputationalContext();
evRes = sotdh.EvaluateAsync(storage, compCont).Result;
result = (double[])sotdh.AggregateAsync(storage, compCont).Result; //val = 6/5*lat = 6/5*(-2.5) = -3
Assert.AreEqual(0.0, result[0], TestConstants.DoublePrecision); //as prate can't be negative, it is coerced to zero
}
public async Task ReanalysisGrowingUncertainty()
{
var storage = TestDataStorageFactory.GetStorageContext(TestConstants.UriReanalysisRegular);
NCEPReanalysisRegularGridDataHandler regularHandler = await NCEPReanalysisRegularGridDataHandler.CreateAsync(storage);
ITimeRegion tr = new TimeRegion(firstYear: 1961, lastYear: 1961, firstDay: 1, lastDay: 1, startHour: 0, stopHour: 0);
IFetchDomain dataDenseDomain = FetchDomain.CreatePoints(
new double[] { 76.25 },
new double[] { 31.25 }, tr);//more dense nodes
IFetchDomain dataSparseDomain = FetchDomain.CreatePoints(
new double[] { 1.25 },
new double[] { 31.25 }, tr);//more rare nodes
FetchRequest dataDenseRequest = new FetchRequest("air", dataDenseDomain);
FetchRequest dataSparseRequest = new FetchRequest("air", dataSparseDomain);
var handlerPrivate = new PrivateObject(regularHandler, new PrivateType(typeof(DataHandlerFacade)));
var evaluatorPrivate = new PrivateObject(handlerPrivate, "uncertaintyEvaluator");
Array dataDenseUncertainty = await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, dataDenseRequest)));
Array dataSparseUncertainty = await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, dataSparseRequest)));
Assert.IsFalse(double.IsNaN((double)dataDenseUncertainty.GetValue(0)));
Assert.IsFalse(double.IsNaN((double)dataSparseUncertainty.GetValue(0)));
Assert.IsTrue(((double)dataDenseUncertainty.GetValue(0)) < double.MaxValue);
Assert.IsTrue(((double)dataSparseUncertainty.GetValue(0)) < double.MaxValue);
Assert.IsTrue(((double)dataDenseUncertainty.GetValue(0)) >= 0.0);
Assert.IsTrue(((double)dataSparseUncertainty.GetValue(0)) >= 0.0);
//Assert.IsTrue(((double)dataDenseUncertainty.GetValue(0))<((double)dataSparseUncertainty.GetValue(0)));
}
public void PartsCountingInSingleDimensionalRequestTest()
{
var tr = new TimeRegion(firstYear: 1990, lastYear: 2001);
var request = new FetchRequest(
"pet",
FetchDomain.CreatePoints(
Enumerable.Range(0, 1500).Select(i => 5.0 + i * 0.1).ToArray(), // 5.0 .. 20.0
Enumerable.Range(0, 1500).Select(i => 70.0 + i * 0.1).ToArray(),
tr));
// 1500 pts in this request
Assert.AreEqual(3, JobManager.GetPartitionsCount(request, 0, 10000, 20));
Assert.AreEqual(1, JobManager.GetPartitionsCount(request, 1000, 10000, 310));
var request2 = new FetchRequest(
"pet",
FetchDomain.CreateCells(
Enumerable.Range(0, 15000).Select(i => 5.0 + i * 0.1).ToArray(),
Enumerable.Range(0, 15000).Select(i => 70.0 + i * 0.1).ToArray(),
Enumerable.Range(0, 15000).Select(i => 6.0 + i * 0.1).ToArray(),
Enumerable.Range(0, 15000).Select(i => 71.0 + i * 0.1).ToArray(),
tr));
//15000 cells
Assert.AreEqual(30, JobManager.GetPartitionsCount(request2, 0, 10000, 102));
Assert.AreEqual(15, JobManager.GetPartitionsCount(request2, 0, 10000, 20));
Assert.AreEqual(15, JobManager.GetPartitionsCount(request2, 1000, 10000, 102));
}
public void MarksRequestsTest()
{
double[] latmin = null, latmax, lonmin, lonmax = null;
int[] startday, stopday, startyear, stopyear, starthour, stophour;
//ClimateService.ServiceUrl = serviceUri;
ClimateService.ServiceUrl = serviceUri;
using (DataSet ds = DataSet.Open(@"Data\423b28bf6a4357f14b64f2b16ab759cb6b5961db.csv?openMode=readOnly"))
{
using (DataSet resultDs = ds.Clone("msds:memory"))
{
latmin = ((double[])ds.Variables["LatMin"].GetData()).Select(e => (double)e).ToArray();
latmax = ((double[])ds.Variables["LatMax"].GetData()).Select(e => (double)e).ToArray();
lonmin = ((double[])ds.Variables["LonMin"].GetData()).Select(e => (double)e).ToArray();
lonmax = ((double[])ds.Variables["LonMax"].GetData()).Select(e => (double)e).ToArray();
startday = ((int[])ds.Variables["StartDay"].GetData());
stopday = ((int[])ds.Variables["StopDay"].GetData());
starthour = ((int[])ds.Variables["StartHour"].GetData());
stophour = ((int[])ds.Variables["StartHour"].GetData());
startyear = ((int[])ds.Variables["StartYear"].GetData());
stopyear = ((int[])ds.Variables["StartYear"].GetData());
TimeRegion tr = new TimeRegion(startyear[0], stopyear[0]);
FetchRequest fr = new FetchRequest("prate", FetchDomain.CreatePoints(latmin, lonmax, tr), new string[] { "CRU CL 2.0" });
var result = ClimateService.FetchAsync(fr).Result;
//resultDs.AddVariable<double>("result",result, "cells_dim");
;
}
}
}
public void PartsCountingTest()
{
var tr = new TimeRegion(firstYear: 2000, lastYear: 2001).GetMonthlyTimeseries();
var request2 = new FetchRequest(
"pet",
FetchDomain.CreateCellGrid(
Enumerable.Range(0, 151).Select(i => 5.0 + i * 0.1).ToArray(), // 5.0 .. 20.0
Enumerable.Range(0, 171).Select(i => 70.0 + i * 0.1).ToArray(),
tr),
new DateTime(2013, 11, 17));
// 306000 pts in this request
Assert.AreEqual(31, JobManager.GetPartitionsCount(request2, 1000, 10000));
Assert.AreEqual(170, JobManager.GetPartitionsCount(request2, 1000, 10000, 310));
Assert.AreEqual(100, JobManager.GetPartitionsCount(request2, 1000, 10000, 100));
var request3 = new FetchRequest(
"pet",
FetchDomain.CreatePointGrid(
Enumerable.Range(0, 50).Select(i => 5.0 + i * 0.1).ToArray(), // 5.0 .. 20.0
Enumerable.Range(0, 170).Select(i => 70.0 + i * 0.1).ToArray(),
tr),
new DateTime(2013, 11, 17));
//102000 pts
Assert.AreEqual(102, JobManager.GetPartitionsCount(request3, 1000, 10000, 102));
}
public async Task MonthlyMeansMissingValueOnWaterTest()
{
var storage = TestDataStorageFactory.GetStorageContext(TestConstants.UriCru);
CruCl20DataHandler cru = await CruCl20DataHandler.CreateAsync(storage);
ITimeRegion tr = new TimeRegion();
IFetchDomain domain = FetchDomain.CreatePoints(
new double[] { 0.5 },
new double[] { 0.5 },
tr);
FetchRequest request = new FetchRequest("tmp", domain);
var handlerPrivate = new PrivateObject(cru, new PrivateType(typeof(DataHandlerFacade)));
var aggregatorPrivate = new PrivateObject(handlerPrivate, "valuesAggregator");
for (int i = 0; i < 2; i++)
{
Array mask = i == 0 ? null : new bool[, ] {
{ true }
};
var res = await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, request), null));
//lon,lat,t
double[] temps = (double[])res;
Assert.AreEqual(1, temps.Length);
Assert.IsTrue(double.IsNaN(temps[0]));
}
}
public void HashTest()
{
TimeRegion tr = new TimeRegion(firstYear: 2000, lastYear: 2001).GetMonthlyTimeseries();
FetchRequest request = new FetchRequest(
"airt",
FetchDomain.CreatePointGrid(
Enumerable.Range(0, 150).Select(i => 50.0 + i * 0.1).ToArray(), // 5.0 .. 20.0
Enumerable.Range(0, 170).Select(i => 30.0 + i * 0.1).ToArray(),
tr),
new DateTime(2012, 11, 13), new string[] { "WorldClim 1.4" });
FetchRequest request2 = new FetchRequest(
"temp",
FetchDomain.CreatePointGrid(
Enumerable.Range(0, 150).Select(i => 50.0 + i * 0.1).ToArray(), // 5.0 .. 20.0
Enumerable.Range(0, 170).Select(i => 30.0 + i * 0.1).ToArray(),
tr),
new DateTime(2012, 11, 13), new string[] { "WorldClim 1.4" });
FetchRequest request3 = new FetchRequest(
"temp",
FetchDomain.CreateCellGrid(
Enumerable.Range(0, 150).Select(i => 50.0 + i * 0.1).ToArray(), // 5.0 .. 20.0
Enumerable.Range(0, 170).Select(i => 30.0 + i * 0.1).ToArray(),
tr),
new DateTime(2012, 11, 13), new string[] { "WorldClim 1.4" });
Assert.IsTrue(request.GetSHAHash() != request2.GetSHAHash());
Assert.IsTrue(request2.GetSHAHash() != request3.GetSHAHash());
}
public async Task LinearWeightsContextFactoryFacadeTest()
{
var lwcff = new LinearWeightsContextFactoryFacade <RealValueNodesStub>(new Stub4(), new Stub3());
var cells = new GeoCellTuple[] { new GeoCellTuple()
{
LatMax = 11.0, LonMin = 13.0, LatMin = 17.0, LonMax = 19.0
} };
var request = new FetchRequest("dummy", FetchDomain.CreateCells(new double[] { 17.0 }, new double[] { 13.0 }, new double[] { 11.0 }, new double[] { 19.0 }, new TimeRegion()));
var storage = TestDataStorageFactory.GetStorage("msds:memory");
var requestContext = RequestContextStub.GetStub(storage, request);
var lwc = await lwcff.CreateAsync(requestContext, cells);
var combs = lwc.Combinations.ToArray();
Assert.AreEqual(1, combs.Length);
Assert.AreEqual(cells[0], combs[0].Item1);
RealValueNodes nodes = combs[0].Item2;
IEnumerable <LinearWeight> weights = combs[0].Item3;
var result = weights.Sum(w => w.Weight * nodes.Values[w.DataIndex]);
Assert.AreEqual(847.0, result);
}
public void CacheTest()
{
if (File.Exists("cache.csv"))
{
File.Delete("cache.csv");
}
var result = ClimateService.FetchAsync(new FetchRequest("airt",
FetchDomain.CreatePoints(new double[] { 57 }, new double[] { 0 }, new TimeRegion())), null, "cache.csv").Result;
Assert.IsTrue(result.Variables.Contains("values"));
try
{
ClimateService.ServiceUrl = "http://notexistentfetchclimateservice.localhost";
var result2 = ClimateService.FetchAsync(new FetchRequest("airt",
FetchDomain.CreatePoints(new double[] { 57 }, new double[] { 0 }, new TimeRegion())), null, "cache.csv").Result;
Assert.IsTrue(result2.Variables.Contains("values"));
try
{
var result3 = ClimateService.FetchAsync(new FetchRequest("airt",
FetchDomain.CreatePoints(new double[] { 57 }, new double[] { 0 }, new TimeRegion()))).Result;
Assert.Fail();
}
catch
{
// It is OK to be here
}
}
finally
{
ClimateService.ServiceUrl = "http://fetchclimate2.cloudapp.net";
}
}
public async Task ReanalysisOutOfDataTest()
{
var storage = TestDataStorageFactory.GetStorageContext(TestConstants.UriReanalysisRegular);
NCEPReanalysisRegularGridDataHandler regularHandler = await NCEPReanalysisRegularGridDataHandler.CreateAsync(storage);
ITimeRegion firstDataYear = new TimeRegion(firstYear: 1245, lastYear: 1245);
IFetchDomain dataDomain = FetchDomain.CreatePoints(
new double[] { -37.5 }, // data index 51
new double[] { 137.5 }, //data index 55
firstDataYear);
FetchRequest dataRequest = new FetchRequest("air", dataDomain);
var handlerPrivate = new PrivateObject(regularHandler, new PrivateType(typeof(DataHandlerFacade)));
var evaluatorPrivate = new PrivateObject(handlerPrivate, "uncertaintyEvaluator");
var aggregatorPrivate = new PrivateObject(handlerPrivate, "valuesAggregator");
Array dataArray = await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, dataRequest), null));;
Array dataUncertainty = await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, dataRequest)));
Assert.AreEqual(1, dataArray.Length);
Assert.AreEqual(1, dataUncertainty.Length);
Assert.IsTrue(double.IsNaN((double)dataArray.GetValue(0)));
Assert.IsTrue(double.IsNaN((double)dataUncertainty.GetValue(0)));
}
public void TestInValidtimeRequest()
{
TimeRegion tr = new TimeRegion(lastDay: 567).GetYearlyTimeseries();
FetchDomain fd = FetchDomain.CreatePointGrid(new double[] { 0, 1, 2 }, new double[] { 3, 4, 5 }, tr);
string error;
Assert.IsFalse(fd.IsContentValid(out error));
Assert.IsFalse(string.IsNullOrEmpty(error));
}
public void TestInValidAxisRangeRequest()
{
TimeRegion tr = new TimeRegion();
FetchDomain fd = FetchDomain.CreatePointGrid(new double[] { 0, 1, 2678 }, new double[] { 3, 4, 5 }, tr);
string error;
Assert.IsFalse(fd.IsContentValid(out error));
Assert.IsFalse(string.IsNullOrEmpty(error));
}
private FetchDomain CreatePointsDomain(TimeRegion tr)
{
var eps = r.NextDouble() / 10.0;
Trace.WriteLine(String.Format("Using eps={0}", eps));
return(FetchDomain.CreatePoints(
new double[] { BayOfBiscaySELat + eps, InFranceLat + eps },
new double[] { BayOfBiscaySELon + eps, InFranceLon + eps },
tr));
}
public async Task MonthlyMeansLandOnlyVariablesTest()
{
var storage = TestDataStorageFactory.GetStorage(TestConstants.UriCru);
CruCl20DataHandler cru = await CruCl20DataHandler.CreateAsync(storage);
TimeRegion tr = new TimeRegion().GetMonthlyTimeseries(firstMonth: 1, lastMonth: 1);
FetchDomain oceanDomain = FetchDomain.CreateCells(
new double[] { 22.0 },
new double[] { -25.0 },
new double[] { 23.0 },
new double[] { -24.0 },
tr);
FetchDomain landDomain = FetchDomain.CreateCells(
new double[] { 24.0 },
new double[] { -10.0 },
new double[] { 25.0 },
new double[] { -9.0 },
tr);
FetchDomain landOceanMixDomain = FetchDomain.CreateCells(
new double[] { 24.0 },
new double[] { -16.0 },
new double[] { 25.0 },
new double[] { -14.0 },
tr);
FetchRequest oceanPrecRequest = new FetchRequest("pre", oceanDomain);
FetchRequest landPrecRequest = new FetchRequest("pre", landDomain);
FetchRequest mixPrecRequest = new FetchRequest("pre", landOceanMixDomain);
var handlerPrivate = new PrivateObject(cru, new PrivateType(typeof(DataHandlerFacade)));
var evaluatorPrivate = new PrivateObject(handlerPrivate, "uncertaintyEvaluator");
var aggregatorPrivate = new PrivateObject(handlerPrivate, "valuesAggregator");
double oceanUnc = (double)(await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, oceanPrecRequest)))).GetValue(0);
double landUnc = (double)(await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, landPrecRequest)))).GetValue(0);
double mixUnc = (double)(await(Task <Array>)(evaluatorPrivate.Invoke("EvaluateAsync", RequestContextStub.GetStub(storage, mixPrecRequest)))).GetValue(0);
Assert.IsTrue(double.IsNaN(oceanUnc));
Assert.AreEqual(double.MaxValue, mixUnc);
Assert.IsTrue(!double.IsNaN(landUnc));
Assert.IsTrue(landUnc < double.MaxValue); //check variogram presence if this fails
double oceanVal = (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, oceanPrecRequest), null))).GetValue(0);
double landVal = (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, landPrecRequest), null))).GetValue(0);
double mixVal = (double)(await(Task <Array>)(aggregatorPrivate.Invoke("AggregateAsync", RequestContextStub.GetStub(storage, mixPrecRequest), null))).GetValue(0);
Assert.IsTrue(double.IsNaN(oceanVal));
Assert.IsTrue(!double.IsNaN(landVal));
Assert.IsTrue(!double.IsNaN(mixVal));
Assert.AreEqual(mixVal, landVal, TestConstants.DoublePrecision);
}
public void PointRequestTest()
{
var tr = new TimeRegion(1990, 2000, 1, 358).GetYearlyTimeseries(1990, 2000, 1, true).GetSeasonlyTimeseries(1, 358, 1, true);
var r = new FetchRequest(
"airt",
FetchDomain.CreatePoints(
new double[] { 50, 52, 54 },
new double[] { 40, 42, 38 },
tr), new string[] { "CRU CL 2.0" });
RemoteFetchClient client = new RemoteFetchClient(new Uri(serviceUri));
var answer = client.FetchAsync(r).Result;
}
public IEnumerable <FetchRequest> GetRequestsForGrid(GeoGrid g)
{
foreach (var pair in variables)
{
yield return(new FetchRequest(
pair.Key,
FetchDomain.CreateCellGrid(g.Latitudes, g.Longitudes, tr),
config.TimeStamp,
pair.Value.ToArray()));
}
yield break;
}
private FetchDomain CreateCellGridDomain(TimeRegion tr)
{
var eps = r.NextDouble() / 10.0;
Trace.WriteLine(String.Format("Using eps={0}", eps));
double latDelta = InFranceLat - BayOfBiscaySELat;
double lonDelta = InFranceLon - BayOfBiscaySELon;
return(FetchDomain.CreateCellGrid(
Enumerable.Range(0, 31).Select(i => eps + BayOfBiscaySELat + i * latDelta / 31.0).ToArray(),
Enumerable.Range(0, 21).Select(i => eps + BayOfBiscaySELon + i * lonDelta / 21.0).ToArray(),
tr));
}
