public float[][] ReadRegionAcrossFiles(int xOffset, int yOffset, int xSize, int ySize)
{
int nPixPerTile = xSize * ySize;
float[][] tileData = new float[m_FileDates.Count][];
int fileNum = 0;
foreach (var t in m_FileDates)
{
var newshape = GDAL_Operations.GetRasterShape(t.Value);
if (newshape.Item1 != Shape.Item1 || newshape.Item2 != Shape.Item2)
{
throw new ArgumentException("Raster shapes don't match");
}
var newGT = GDAL_Operations.GetGeoTransform(t.Value);
if (!GeoTransform.SequenceEqual(newGT))
{
throw new ArgumentException("Raster geotransforms don't match");
}
var newNDV = GDAL_Operations.GetNoDataValue(t.Value);
if (newNDV != NoDataValue)
{
throw new ArgumentException("Raster nodata values don't match");
}
var newProj = GDAL_Operations.GetProjection(t.Value);
if (newProj != Projection)
{
throw new ArgumentException("Raster projections don't match");
}
tileData[fileNum] = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
t.Value, xSize, ySize, xOffset, yOffset, 1);
fileNum += 1;
}
return(tileData);
}
static float[] TestReadTileAcrossTime(int column, int row, int xSize = 512, int ySize = 512)
{
int xOff = column * xSize;
int yOff = row * ySize;
string fileWildCard = "F:\\MOD11A2_Gapfilled_Output\\LST_Day\\Output_Final_30k_2030pc\\*Data.tif";
IFilenameDateParser modisFileParse = new FilenameDateParser_MODIS8DayRaw();
var details = GetFilenamesAndDates(fileWildCard, modisFileParse);
string firstFileName = details[0].Item1;
double[] overallGT = GDAL_Operations.GetGeoTransform(firstFileName);
var shape = GDAL_Operations.GetRasterShape(firstFileName);
if (yOff > shape.Item1 || xOff > shape.Item2)
{
throw new ArgumentException("you specified a column or row greater than the number of tiles available");
}
if (yOff + ySize > shape.Item1)
{
ySize = shape.Item1 - yOff;
}
if (xOff + xSize > shape.Item2)
{
xSize = shape.Item2 - xOff;
}
int nPix = xSize * ySize;
float[] tileData = new float[nPix * details.Count];
for (int t = 0; t < details.Count; t++)
{
int pxStart = nPix * t;
string filename = details[t].Item1;
DateTime filedate = details[t].Item2;
var newshape = GDAL_Operations.GetRasterShape(filename);
if (newshape.Item1 != shape.Item1 || newshape.Item2 != shape.Item2)
{
throw new ArgumentException("Raster shapes don't match");
}
var bandarr = GDAL_Operations.ReadGDALRasterBandsToFlatArray(filename, xSize, ySize, xOff, yOff, 1);
Array.Copy(bandarr, 0, tileData, pxStart, bandarr.Length);
// tileData[pxStart : pxStart+nPix] = arr;
}
return(tileData);
}
public float[][] ReadRegionAcrossFiles_MP(int xOffset, int yOffset, int xSize, int ySize)
{
int nPixPerTile = xSize * ySize;
float[][] tileData = new float[m_FileDates.Count][];
ParallelOptions b = new ParallelOptions();
b.MaxDegreeOfParallelism = 6;
var keys = m_FileDates.Keys;
Parallel.For(0, keys.Count, b, c =>
{
var fDate = keys[c];
var fName = m_FileDates[fDate];
var newshape = GDAL_Operations.GetRasterShape(fName);
if (newshape.Item1 != Shape.Item1 || newshape.Item2 != Shape.Item2)
{
throw new ArgumentException("Raster shapes don't match");
}
var newGT = GDAL_Operations.GetGeoTransform(fName);
if (!GeoTransform.SequenceEqual(newGT))
{
throw new ArgumentException("Raster geotransforms don't match");
}
var newNDV = GDAL_Operations.GetNoDataValue(fName);
if (newNDV != NoDataValue)
{
throw new ArgumentException("Raster nodata values don't match");
}
var newProj = GDAL_Operations.GetProjection(fName);
if (newProj != Projection)
{
throw new ArgumentException("Raster projections don't match");
}
tileData[c] = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
fName, xSize, ySize, xOffset, yOffset, 1);
});
return(tileData);
}
static void Main(string [] args)
{
Environment.SetEnvironmentVariable("PATH", Environment.GetEnvironmentVariable("PATH")
+ ";C:\\Users\\zool1301.NDPH\\Documents\\Code_General\\temp-suitability\\TempSuitability_CSharp\\packages\\GDAL.Native.1.11.1\\gdal\\x64");
//string testFile = "G:\\DataPrep\\ts_global\\TempSuitability.Pf.AnnualInfectiousDays.1k.2010.global.tif";
string testFile = "F:\\MOD11A2_Gapfilled_Output\\LST_Day\\Output_Final_30k_2030pc\\LST_Day_All.vrt";
int xsize = 512;
int ysize = 512;
int nbands = 727;
Stopwatch sw = new Stopwatch();
sw.Start();
//var arr = GDAL_Operations.ReadGDALRasterBandsToFlatArray(testFile, xsize, ysize, 20480, 9216, null);
var arr = TestReadTileAcrossTime(40, 18);
sw.Stop();
Console.WriteLine("Time elapsed reading tif data into flat array via pointers: {0}", sw.Elapsed);
sw.Restart();
int nPx = xsize * ysize;
var cellArr = new float[nPx][];
for (int y = 0; y < ysize; y++)
{
for (int x = 0; x < xsize; x++)
{
int cellNum = y * xsize + x;
cellArr[cellNum] = new float[nbands];
for (int z = 0; z < nbands; z++)
{
cellArr[cellNum][z] = arr[nPx * z + cellNum];
}
}
}
Console.WriteLine("Time elapsed reformatting flat data into cell-band order: {0}", sw.Elapsed);
arr = null;
sw.Restart();
var arr3 = GDAL_Operations.ReadGDALRasterBandsToFlatArray(testFile, xsize, ysize, 20480, 9216, null);
sw.Stop();
Console.WriteLine("Time elapsed reading vrt data into flat array via pointers: {0}", sw.Elapsed);
sw.Restart();
var cellArr3 = new float[nPx][];
for (int y = 0; y < ysize; y++)
{
for (int x = 0; x < xsize; x++)
{
int cellNum = y * xsize + x;
cellArr3[cellNum] = new float[nbands];
for (int z = 0; z < nbands; z++)
{
cellArr3[cellNum][z] = arr3[nPx * z + cellNum];
}
}
}
Console.WriteLine("Time elapsed reformatting flat vrt data into cell-band order: {0}", sw.Elapsed);
arr3 = null;
sw.Restart();
var arr2 = GDAL_Operations.ReadGDALRasterBandsToJaggedArray(testFile, xsize, ysize, 20480, 9216, null);
sw.Stop();
Console.WriteLine("Time elapsed reading data into jagged array via pointers: {0}", sw.Elapsed);
sw.Restart();
var cellArr2 = new float[xsize * ysize][];
for (int y = 0; y < ysize; y++)
{
for (int x = 0; x < xsize; x++)
{
int cellNum = y * xsize + x;
cellArr2[cellNum] = new float[nbands];
for (int z = 0; z < nbands; z++)
{
cellArr2[cellNum][z] = arr2[z][cellNum];
}
}
}
Console.WriteLine("Time elapsed reformatting jagged data into cell-band order: {0}", sw.Elapsed);
System.Console.ReadKey();
}
/// <summary>
/// Runs a temperature suitability model for all pixels in a region specified by pixel limits.
/// Output is a jagged array with one value for each cell starting at top left and then row by
/// row to bottom right, EXCEPT if no pixel in the tile is in a data area in which case the output
/// is an array with length zero.
/// Otherwise, each value is an array with one TS value for each month of the run period,
/// EXCEPT if the cell is in the sea / masked area, in which case it is an array of length 0.
/// Each cell location is done independently and this is therefore multithreaded.
/// </summary>
/// <param name="xOff"></param>
/// <param name="yOff"></param>
/// <param name="xSize"></param>
/// <param name="ySize"></param>
/// <returns></returns>
public float[][] RunTile(int xOff, int yOff, int xSize, int ySize)
{
// test area: int xOff=20480, int yOff=9216, int xSize=512, int ySize=512
var lsMask = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
_cfg.dataPathConfig.MaskFile,
xSize, ySize, xOff, yOff, 1);
if (!lsMask.Any(v => v == _cfg.modelRunConfig.MaskValidValue))
{
// this whole tile is in the sea, no need to run, return as a special case a zero length cell array
return(new float[0][]);
}
var maxTempData = _maxReader.ReadCellData(xOff, yOff, xSize, ySize);
var lats = _maxReader.GetSubsetLatitudeCoords(yOff, ySize);
var lons = _maxReader.GetSubsetLongitudeCoords(xOff, xSize);
var minTempData = _minReader.ReadCellData(xOff, yOff, xSize, ySize);
int numCells = xSize * ySize;
// there's no reason in principle why we couldn't allow different numbers of min and max temp files although of
// course we'd have to pass separate dates arrays into the model then, but hey. It would be a bit more effort
// to handle different geotransforms in the max and min temps data and this way we make it less likely that
// they've come from separate sources.
Debug.Assert(minTempData.Length == maxTempData.Length);
Debug.Assert(maxTempData.Length == numCells);
var dates = _maxReader.Filedates.ToArray();
var nFiles = dates.Length;
// get the model parameters from the default settings file, bearing in mind that this could have been re-set
// at initialisation to a file specified on the commandline, rather than just being the default app config
//var set = Properties.Settings.Default;
var set = _cfg.modelConfig;
PopulationParams popParams = new PopulationParams();
popParams.DegreeDayThreshold = set.SporogenesisDegreeDays;
popParams.MinTempThreshold = set.MinTempThresholdCelsius;
popParams.MosquitoDeathTemperature = set.DeathTempCelsius;;
popParams.MosquitoLifespanDays = new TimeSpan((int)set.LifespanDays, 0, 0, 0);;
popParams.SliceLength = new TimeSpan((int)set.SliceLengthHours, 0, 0);
popParams.MaxTempSuitability = set.MaxTSNormaliser;
// max ts for default settings is 34.2467; the Weiss code had 33.89401 , not sure how generated.
// I got this using iterative solver in excel.
Console.WriteLine("Tile data loaded, computation beginning");
var mConfig = _cfg.modelRunConfig;
float[][] tsOut = new float[numCells][];
DateTime[] outputDates = null;
ParallelOptions b = new ParallelOptions();
if (mConfig.MaxThreads != 0)
{
// set threads to 1 for easier step-through debugging or some other number to not hog the whole machine
b.MaxDegreeOfParallelism = (int)mConfig.MaxThreads;
//System.Threading.ThreadPool.SetMaxThreads(set.MaxThreads, set.MaxThreads);
//System.Threading.ThreadPool.SetMinThreads(set.MinThreads, set.MinThreads);
}
int testnum = 0;
while (outputDates == null && testnum < numCells)
{
// if we haven't got at least 50% of the data and 100+ points it's probably crap
// (unless we're playing with synoptic data or something, so make this threshold configurable).
// This doesn't affect the date calculation but the spline will throw an error
// on initialisation
var nValid = Math.Min(
maxTempData[testnum].Count(v => v != _maxReader.NoDataValue),
minTempData[testnum].Count(v => v != _minReader.NoDataValue));
if (nValid < nFiles / 2 || nValid < mConfig.MinRequiredDataPoints)
{
testnum += 1;
continue;
}
// set up a dummy model purely to parse the output dates (that they will all share)
// avoids the need to test for the first one to do this in the parallel loop, which needs a lock,
// which slows things right down with >>20 cores
TSCellModel tsModel = new TSCellModel(
popParams,
new GeographicCellLocation()
{
Latitude = lats[0], Longitude = lons[0]
},
PopulationTypes.Pointers);
if (!tsModel.SetData(maxTempData[testnum], minTempData[testnum], dates, _maxReader.NoDataValue.Value,
mConfig.MaxTempFilesAreLST, mConfig.MinTempFilesAreLST))
{
throw new ApplicationException("Pop goes the weasel");
}
;
outputDates = tsModel.OutputDates;
break;
}
Parallel.For(0, numCells, b, c =>
{
if (lsMask[c] != mConfig.MaskValidValue)
{
// this cell is in the sea, no need to run, return as a special case a zero length result array
tsOut[c] = new float[0];
}
else
{
int rownum = c / xSize;
int colnum = c % xSize;
GeographicCellLocation geogParams = new GeographicCellLocation();
geogParams.Latitude = lats[rownum];
geogParams.Longitude = lons[colnum];
//geogParams.ModelRuntimeDays = nDays;
//geogParams.StartJulianDay = 0;
// run only if we've got at least 50% of the data and 100+ points (or whatever is configured)
var nValid = Math.Min(
maxTempData[c].Count(v => v != _maxReader.NoDataValue),
minTempData[c].Count(v => v != _minReader.NoDataValue));
if (nValid < nFiles / 2 || nValid < mConfig.MinRequiredDataPoints)
{
tsOut[c] = new float[0];
}
else
{
TSCellModel tsModel = new TSCellModel(popParams, geogParams, PopulationTypes.Pointers);
// var dd = dayData[c];
// var nd = nightData[c];
// float optimal = 28.6857194664029F;
// for (int i = 0; i < 727; i++) {
// dd[i] = optimal;
// if (i%3==0) { dd[i] = 43; }
// nd[i] = optimal; }
//tsModel.SetData(dd, nd, dates, _maxReader.NoDataValue.Value, false);
if (!tsModel.SetData(maxTempData[c], minTempData[c], dates, _maxReader.NoDataValue.Value,
mConfig.MaxTempFilesAreLST, mConfig.MinTempFilesAreLST))
{
throw new ApplicationException("Pop goes the weasel");
}
;
// run the entire ts model for this location
float[] tsCell = tsModel.RunModel();
int nOutputPoints = tsCell.Length;
tsOut[c] = tsCell;
/*lock (_lockobj) // ensure only 1 thread makes this check
* {
* // calculate and record the dates of the outputs for an arbitrary one of the cell models
* if (outputDates == null)
* {
* outputDates = tsModel.OutputDates;
* }
* }*/
}
}
}
);
if (_outputDates == null)
{
_outputDates = outputDates;
}
else if (outputDates != null && !_outputDates.SequenceEqual(outputDates))
{
throw new ApplicationException("Dates have changed between tiles somehow, this shouldn't happen...");
}
return(tsOut);
}
