static double[][] RunForCoarseStripe(string FilePattern, PopulationTypes ModelType)
{
string fnPattern = "C:\\Users\\zool1301.NDPH\\Documents\\Code_General\\temp-suitability\\Original\\c_code\\test5\\data\\{0}.{1}.dat";
string latFile = String.Format(fnPattern, "lat", FilePattern);
string lonFile = String.Format(fnPattern, "long", FilePattern);
string minFile = String.Format(fnPattern, "tmin", FilePattern);
string maxFile = String.Format(fnPattern, "tmax", FilePattern);
double[] tLats = File.ReadAllLines(latFile).
Select(l => double.Parse(l)).ToArray();
double[] tLongs = File.ReadAllLines(lonFile).
Select(l => double.Parse(l)).ToArray();
double[] tMins = File.ReadAllLines(minFile).
Select(l => double.Parse(l)).ToArray();
double[] tMaxs = File.ReadAllLines(maxFile).
Select(l => double.Parse(l)).ToArray();
int numCells = tLats.Length;
int nDays = tMins.Length / tLats.Length;
PopulationParams popParams = new PopulationParams();
popParams.DegreeDayThreshold = 111;
popParams.MinTempThreshold = 11;
popParams.MosquitoLifespanDays = new TimeSpan(31, 0, 0, 0);
popParams.SliceLength = new TimeSpan(2, 0, 0);
double[][] tsOut = new double[numCells][];
//int c = 0;
int numBlocks = 200;
IEnumerable <int> blocks = Enumerable.Range(0, numBlocks);
int cellsPerBlock = numCells / numBlocks;
Parallel.ForEach(blocks, blockID =>
{
int blockCellStartIdx = blockID * cellsPerBlock;
for (int c = blockCellStartIdx; (c < (blockCellStartIdx + cellsPerBlock) && c < numCells); c++)
{
tsOut[c] = new double[nDays];
double lat = tLats[c];
double lon = tLongs[c];
int startIdx = c * nDays;
int endIdx = startIdx + nDays;
double[] tMinsForCell = new double[nDays];
double[] tMaxsForCell = new double[nDays];
Array.Copy(tMins, startIdx, tMinsForCell, 0, nDays);
Array.Copy(tMaxs, startIdx, tMaxsForCell, 0, nDays);
GeographicCellLocation geogParams = new GeographicCellLocation();
geogParams.Latitude = tLats[c];
geogParams.Longitude = tLongs[c];
TSCellModel tsModel = new TSCellModel(popParams, geogParams, ModelType);
//double[] tsCell = tsModel.RunModel(tMinsForCell, tMaxsForCell);
//for (int i = 0; i < nDays; i++)
//{
// tsOut[c][i] = tsCell[i];
//}
}
});
return(tsOut);
}
/// <summary>
/// Runs a temperature suitability model for all pixels in a region specified by pixel limits.
/// Output is a jagged array with one value for each cell starting at top left and then row by
/// row to bottom right, EXCEPT if no pixel in the tile is in a data area in which case the output
/// is an array with length zero.
/// Otherwise, each value is an array with one TS value for each month of the run period,
/// EXCEPT if the cell is in the sea / masked area, in which case it is an array of length 0.
/// Each cell location is done independently and this is therefore multithreaded.
/// </summary>
/// <param name="xOff"></param>
/// <param name="yOff"></param>
/// <param name="xSize"></param>
/// <param name="ySize"></param>
/// <returns></returns>
public float[][] RunTile(int xOff, int yOff, int xSize, int ySize)
{
// test area: int xOff=20480, int yOff=9216, int xSize=512, int ySize=512
var lsMask = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
_cfg.dataPathConfig.MaskFile,
xSize, ySize, xOff, yOff, 1);
if (!lsMask.Any(v => v == _cfg.modelRunConfig.MaskValidValue))
{
// this whole tile is in the sea, no need to run, return as a special case a zero length cell array
return(new float[0][]);
}
var maxTempData = _maxReader.ReadCellData(xOff, yOff, xSize, ySize);
var lats = _maxReader.GetSubsetLatitudeCoords(yOff, ySize);
var lons = _maxReader.GetSubsetLongitudeCoords(xOff, xSize);
var minTempData = _minReader.ReadCellData(xOff, yOff, xSize, ySize);
int numCells = xSize * ySize;
// there's no reason in principle why we couldn't allow different numbers of min and max temp files although of
// course we'd have to pass separate dates arrays into the model then, but hey. It would be a bit more effort
// to handle different geotransforms in the max and min temps data and this way we make it less likely that
// they've come from separate sources.
Debug.Assert(minTempData.Length == maxTempData.Length);
Debug.Assert(maxTempData.Length == numCells);
var dates = _maxReader.Filedates.ToArray();
var nFiles = dates.Length;
// get the model parameters from the default settings file, bearing in mind that this could have been re-set
// at initialisation to a file specified on the commandline, rather than just being the default app config
//var set = Properties.Settings.Default;
var set = _cfg.modelConfig;
PopulationParams popParams = new PopulationParams();
popParams.DegreeDayThreshold = set.SporogenesisDegreeDays;
popParams.MinTempThreshold = set.MinTempThresholdCelsius;
popParams.MosquitoDeathTemperature = set.DeathTempCelsius;;
popParams.MosquitoLifespanDays = new TimeSpan((int)set.LifespanDays, 0, 0, 0);;
popParams.SliceLength = new TimeSpan((int)set.SliceLengthHours, 0, 0);
popParams.MaxTempSuitability = set.MaxTSNormaliser;
// max ts for default settings is 34.2467; the Weiss code had 33.89401 , not sure how generated.
// I got this using iterative solver in excel.
Console.WriteLine("Tile data loaded, computation beginning");
var mConfig = _cfg.modelRunConfig;
float[][] tsOut = new float[numCells][];
DateTime[] outputDates = null;
ParallelOptions b = new ParallelOptions();
if (mConfig.MaxThreads != 0)
{
// set threads to 1 for easier step-through debugging or some other number to not hog the whole machine
b.MaxDegreeOfParallelism = (int)mConfig.MaxThreads;
//System.Threading.ThreadPool.SetMaxThreads(set.MaxThreads, set.MaxThreads);
//System.Threading.ThreadPool.SetMinThreads(set.MinThreads, set.MinThreads);
}
int testnum = 0;
while (outputDates == null && testnum < numCells)
{
// if we haven't got at least 50% of the data and 100+ points it's probably crap
// (unless we're playing with synoptic data or something, so make this threshold configurable).
// This doesn't affect the date calculation but the spline will throw an error
// on initialisation
var nValid = Math.Min(
maxTempData[testnum].Count(v => v != _maxReader.NoDataValue),
minTempData[testnum].Count(v => v != _minReader.NoDataValue));
if (nValid < nFiles / 2 || nValid < mConfig.MinRequiredDataPoints)
{
testnum += 1;
continue;
}
// set up a dummy model purely to parse the output dates (that they will all share)
// avoids the need to test for the first one to do this in the parallel loop, which needs a lock,
// which slows things right down with >>20 cores
TSCellModel tsModel = new TSCellModel(
popParams,
new GeographicCellLocation()
{
Latitude = lats[0], Longitude = lons[0]
},
PopulationTypes.Pointers);
if (!tsModel.SetData(maxTempData[testnum], minTempData[testnum], dates, _maxReader.NoDataValue.Value,
mConfig.MaxTempFilesAreLST, mConfig.MinTempFilesAreLST))
{
throw new ApplicationException("Pop goes the weasel");
}
;
outputDates = tsModel.OutputDates;
break;
}
Parallel.For(0, numCells, b, c =>
{
if (lsMask[c] != mConfig.MaskValidValue)
{
// this cell is in the sea, no need to run, return as a special case a zero length result array
tsOut[c] = new float[0];
}
else
{
int rownum = c / xSize;
int colnum = c % xSize;
GeographicCellLocation geogParams = new GeographicCellLocation();
geogParams.Latitude = lats[rownum];
geogParams.Longitude = lons[colnum];
//geogParams.ModelRuntimeDays = nDays;
//geogParams.StartJulianDay = 0;
// run only if we've got at least 50% of the data and 100+ points (or whatever is configured)
var nValid = Math.Min(
maxTempData[c].Count(v => v != _maxReader.NoDataValue),
minTempData[c].Count(v => v != _minReader.NoDataValue));
if (nValid < nFiles / 2 || nValid < mConfig.MinRequiredDataPoints)
{
tsOut[c] = new float[0];
}
else
{
TSCellModel tsModel = new TSCellModel(popParams, geogParams, PopulationTypes.Pointers);
// var dd = dayData[c];
// var nd = nightData[c];
// float optimal = 28.6857194664029F;
// for (int i = 0; i < 727; i++) {
// dd[i] = optimal;
// if (i%3==0) { dd[i] = 43; }
// nd[i] = optimal; }
//tsModel.SetData(dd, nd, dates, _maxReader.NoDataValue.Value, false);
if (!tsModel.SetData(maxTempData[c], minTempData[c], dates, _maxReader.NoDataValue.Value,
mConfig.MaxTempFilesAreLST, mConfig.MinTempFilesAreLST))
{
throw new ApplicationException("Pop goes the weasel");
}
;
// run the entire ts model for this location
float[] tsCell = tsModel.RunModel();
int nOutputPoints = tsCell.Length;
tsOut[c] = tsCell;
/*lock (_lockobj) // ensure only 1 thread makes this check
* {
* // calculate and record the dates of the outputs for an arbitrary one of the cell models
* if (outputDates == null)
* {
* outputDates = tsModel.OutputDates;
* }
* }*/
}
}
}
);
if (_outputDates == null)
{
_outputDates = outputDates;
}
else if (outputDates != null && !_outputDates.SequenceEqual(outputDates))
{
throw new ApplicationException("Dates have changed between tiles somehow, this shouldn't happen...");
}
return(tsOut);
}