/// <summary>
/// Parse key information about the first raster: the geotransform, shape (n pixels X and Y), and
/// nodata value (of the first band). Also calculate the latitude of each row and the longitude of
/// each column.
/// </summary>
private void PopulateRasterProperties()
{
string firstFilename = m_FileDates.First().Value;
GeoTransform = GDAL_Operations.GetGeoTransform(firstFilename);
Shape = GDAL_Operations.GetRasterShape(firstFilename);
NoDataValue = GDAL_Operations.GetNoDataValue(firstFilename);
Projection = GDAL_Operations.GetProjection(firstFilename);
double[] latcoords = new double[Shape.Item1];
double[] loncoords = new double[Shape.Item2];
double originX = GeoTransform[0];
double cellsizeX = GeoTransform[1];
double originY = GeoTransform[3];
double cellsizeY = GeoTransform[5];
for (int i = 0; i < Shape.Item1; i++)
{
latcoords[i] = originY + i * cellsizeY;
}
for (int i = 0; i < Shape.Item2; i++)
{
loncoords[i] = originX + i * cellsizeX;
}
GlobalLatCoords = latcoords;
GlobalLonCoords = loncoords;
}
public float[][] ReadRegionAcrossFiles(int xOffset, int yOffset, int xSize, int ySize)
{
int nPixPerTile = xSize * ySize;
float[][] tileData = new float[m_FileDates.Count][];
int fileNum = 0;
foreach (var t in m_FileDates)
{
var newshape = GDAL_Operations.GetRasterShape(t.Value);
if (newshape.Item1 != Shape.Item1 || newshape.Item2 != Shape.Item2)
{
throw new ArgumentException("Raster shapes don't match");
}
var newGT = GDAL_Operations.GetGeoTransform(t.Value);
if (!GeoTransform.SequenceEqual(newGT))
{
throw new ArgumentException("Raster geotransforms don't match");
}
var newNDV = GDAL_Operations.GetNoDataValue(t.Value);
if (newNDV != NoDataValue)
{
throw new ArgumentException("Raster nodata values don't match");
}
var newProj = GDAL_Operations.GetProjection(t.Value);
if (newProj != Projection)
{
throw new ArgumentException("Raster projections don't match");
}
tileData[fileNum] = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
t.Value, xSize, ySize, xOffset, yOffset, 1);
fileNum += 1;
}
return(tileData);
}
public double[] GetSubsetGeoTransform(PixelLims TileCoords)
{
string firstFilename = m_FileDates.First().Value;
var res = GDAL_Operations.GetClippedGeoTransform(firstFilename, TileCoords);
return(res);
}
static float[] TestReadTileAcrossTime(int column, int row, int xSize = 512, int ySize = 512)
{
int xOff = column * xSize;
int yOff = row * ySize;
string fileWildCard = "F:\\MOD11A2_Gapfilled_Output\\LST_Day\\Output_Final_30k_2030pc\\*Data.tif";
IFilenameDateParser modisFileParse = new FilenameDateParser_MODIS8DayRaw();
var details = GetFilenamesAndDates(fileWildCard, modisFileParse);
string firstFileName = details[0].Item1;
double[] overallGT = GDAL_Operations.GetGeoTransform(firstFileName);
var shape = GDAL_Operations.GetRasterShape(firstFileName);
if (yOff > shape.Item1 || xOff > shape.Item2)
{
throw new ArgumentException("you specified a column or row greater than the number of tiles available");
}
if (yOff + ySize > shape.Item1)
{
ySize = shape.Item1 - yOff;
}
if (xOff + xSize > shape.Item2)
{
xSize = shape.Item2 - xOff;
}
int nPix = xSize * ySize;
float[] tileData = new float[nPix * details.Count];
for (int t = 0; t < details.Count; t++)
{
int pxStart = nPix * t;
string filename = details[t].Item1;
DateTime filedate = details[t].Item2;
var newshape = GDAL_Operations.GetRasterShape(filename);
if (newshape.Item1 != shape.Item1 || newshape.Item2 != shape.Item2)
{
throw new ArgumentException("Raster shapes don't match");
}
var bandarr = GDAL_Operations.ReadGDALRasterBandsToFlatArray(filename, xSize, ySize, xOff, yOff, 1);
Array.Copy(bandarr, 0, tileData, pxStart, bandarr.Length);
// tileData[pxStart : pxStart+nPix] = arr;
}
return(tileData);
}
// see also https://www.codeproject.com/Articles/14465/Specify-a-Configuration-File-at-Runtime-for-a-C-Co
private TSModelRunner(IFilenameDateParser _parser, TSIModelConfig cfg)
{
_fnParser = _parser;
//var set = Properties.Settings.Default;
//System.Console.WriteLine("Looking for files in " + m_FileWildCard);
//var d = new System.Diagnostics.DefaultTraceListener();
_maxReader = new TiffCubeReader(cfg.dataPathConfig.MaxTempFiles, _fnParser,
cfg.modelRunConfig.ReadFromDate, cfg.modelRunConfig.ReadToDate);
var nMax = _maxReader.Filenames.Count;
Console.WriteLine("Looking for max temp files in " + cfg.dataPathConfig.MaxTempFiles +
" - found " + nMax.ToString());
_minReader = new TiffCubeReader(cfg.dataPathConfig.MinTempFiles, _fnParser,
cfg.modelRunConfig.ReadFromDate, cfg.modelRunConfig.ReadToDate);
var nMin = _minReader.Filenames.Count;
Console.WriteLine("Looking for min temp files in " + cfg.dataPathConfig.MinTempFiles +
" - found " + nMin.ToString());
if (nMax == 0 || nMin == 0)
{
throw new ArgumentException("Can't continue without data");
// I don't see any reason not to continue if the numbers for day and night aren't actually equal
}
if (!_maxReader.GeoTransform.SequenceEqual(_minReader.GeoTransform))
{
throw new ArgumentException("max and min temp image transforms do not match!");
}
if (!GDAL_Operations.GetGeoTransform(cfg.dataPathConfig.MaskFile).SequenceEqual(_maxReader.GeoTransform))
{
throw new ArgumentException("Land-sea mask doesn't match images!");
}
if (!System.IO.Directory.Exists(cfg.dataPathConfig.OutputFolder))
{
System.IO.Directory.CreateDirectory(cfg.dataPathConfig.OutputFolder);
}
_cfg = cfg;
}
public float[][] ReadRegionAcrossFiles_MP(int xOffset, int yOffset, int xSize, int ySize)
{
int nPixPerTile = xSize * ySize;
float[][] tileData = new float[m_FileDates.Count][];
ParallelOptions b = new ParallelOptions();
b.MaxDegreeOfParallelism = 6;
var keys = m_FileDates.Keys;
Parallel.For(0, keys.Count, b, c =>
{
var fDate = keys[c];
var fName = m_FileDates[fDate];
var newshape = GDAL_Operations.GetRasterShape(fName);
if (newshape.Item1 != Shape.Item1 || newshape.Item2 != Shape.Item2)
{
throw new ArgumentException("Raster shapes don't match");
}
var newGT = GDAL_Operations.GetGeoTransform(fName);
if (!GeoTransform.SequenceEqual(newGT))
{
throw new ArgumentException("Raster geotransforms don't match");
}
var newNDV = GDAL_Operations.GetNoDataValue(fName);
if (newNDV != NoDataValue)
{
throw new ArgumentException("Raster nodata values don't match");
}
var newProj = GDAL_Operations.GetProjection(fName);
if (newProj != Projection)
{
throw new ArgumentException("Raster projections don't match");
}
tileData[c] = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
fName, xSize, ySize, xOffset, yOffset, 1);
});
return(tileData);
}
static void Main(string [] args)
{
Environment.SetEnvironmentVariable("PATH", Environment.GetEnvironmentVariable("PATH")
+ ";C:\\Users\\zool1301.NDPH\\Documents\\Code_General\\temp-suitability\\TempSuitability_CSharp\\packages\\GDAL.Native.1.11.1\\gdal\\x64");
//string testFile = "G:\\DataPrep\\ts_global\\TempSuitability.Pf.AnnualInfectiousDays.1k.2010.global.tif";
string testFile = "F:\\MOD11A2_Gapfilled_Output\\LST_Day\\Output_Final_30k_2030pc\\LST_Day_All.vrt";
int xsize = 512;
int ysize = 512;
int nbands = 727;
Stopwatch sw = new Stopwatch();
sw.Start();
//var arr = GDAL_Operations.ReadGDALRasterBandsToFlatArray(testFile, xsize, ysize, 20480, 9216, null);
var arr = TestReadTileAcrossTime(40, 18);
sw.Stop();
Console.WriteLine("Time elapsed reading tif data into flat array via pointers: {0}", sw.Elapsed);
sw.Restart();
int nPx = xsize * ysize;
var cellArr = new float[nPx][];
for (int y = 0; y < ysize; y++)
{
for (int x = 0; x < xsize; x++)
{
int cellNum = y * xsize + x;
cellArr[cellNum] = new float[nbands];
for (int z = 0; z < nbands; z++)
{
cellArr[cellNum][z] = arr[nPx * z + cellNum];
}
}
}
Console.WriteLine("Time elapsed reformatting flat data into cell-band order: {0}", sw.Elapsed);
arr = null;
sw.Restart();
var arr3 = GDAL_Operations.ReadGDALRasterBandsToFlatArray(testFile, xsize, ysize, 20480, 9216, null);
sw.Stop();
Console.WriteLine("Time elapsed reading vrt data into flat array via pointers: {0}", sw.Elapsed);
sw.Restart();
var cellArr3 = new float[nPx][];
for (int y = 0; y < ysize; y++)
{
for (int x = 0; x < xsize; x++)
{
int cellNum = y * xsize + x;
cellArr3[cellNum] = new float[nbands];
for (int z = 0; z < nbands; z++)
{
cellArr3[cellNum][z] = arr3[nPx * z + cellNum];
}
}
}
Console.WriteLine("Time elapsed reformatting flat vrt data into cell-band order: {0}", sw.Elapsed);
arr3 = null;
sw.Restart();
var arr2 = GDAL_Operations.ReadGDALRasterBandsToJaggedArray(testFile, xsize, ysize, 20480, 9216, null);
sw.Stop();
Console.WriteLine("Time elapsed reading data into jagged array via pointers: {0}", sw.Elapsed);
sw.Restart();
var cellArr2 = new float[xsize * ysize][];
for (int y = 0; y < ysize; y++)
{
for (int x = 0; x < xsize; x++)
{
int cellNum = y * xsize + x;
cellArr2[cellNum] = new float[nbands];
for (int z = 0; z < nbands; z++)
{
cellArr2[cellNum][z] = arr2[z][cellNum];
}
}
}
Console.WriteLine("Time elapsed reformatting jagged data into cell-band order: {0}", sw.Elapsed);
System.Console.ReadKey();
}
/// <summary>
/// Runs a temperature suitability model for all pixels in a region specified by pixel limits.
/// Output is a jagged array with one value for each cell starting at top left and then row by
/// row to bottom right, EXCEPT if no pixel in the tile is in a data area in which case the output
/// is an array with length zero.
/// Otherwise, each value is an array with one TS value for each month of the run period,
/// EXCEPT if the cell is in the sea / masked area, in which case it is an array of length 0.
/// Each cell location is done independently and this is therefore multithreaded.
/// </summary>
/// <param name="xOff"></param>
/// <param name="yOff"></param>
/// <param name="xSize"></param>
/// <param name="ySize"></param>
/// <returns></returns>
public float[][] RunTile(int xOff, int yOff, int xSize, int ySize)
{
// test area: int xOff=20480, int yOff=9216, int xSize=512, int ySize=512
var lsMask = GDAL_Operations.ReadGDALRasterBandsToFlatArray(
_cfg.dataPathConfig.MaskFile,
xSize, ySize, xOff, yOff, 1);
if (!lsMask.Any(v => v == _cfg.modelRunConfig.MaskValidValue))
{
// this whole tile is in the sea, no need to run, return as a special case a zero length cell array
return(new float[0][]);
}
var maxTempData = _maxReader.ReadCellData(xOff, yOff, xSize, ySize);
var lats = _maxReader.GetSubsetLatitudeCoords(yOff, ySize);
var lons = _maxReader.GetSubsetLongitudeCoords(xOff, xSize);
var minTempData = _minReader.ReadCellData(xOff, yOff, xSize, ySize);
int numCells = xSize * ySize;
// there's no reason in principle why we couldn't allow different numbers of min and max temp files although of
// course we'd have to pass separate dates arrays into the model then, but hey. It would be a bit more effort
// to handle different geotransforms in the max and min temps data and this way we make it less likely that
// they've come from separate sources.
Debug.Assert(minTempData.Length == maxTempData.Length);
Debug.Assert(maxTempData.Length == numCells);
var dates = _maxReader.Filedates.ToArray();
var nFiles = dates.Length;
// get the model parameters from the default settings file, bearing in mind that this could have been re-set
// at initialisation to a file specified on the commandline, rather than just being the default app config
//var set = Properties.Settings.Default;
var set = _cfg.modelConfig;
PopulationParams popParams = new PopulationParams();
popParams.DegreeDayThreshold = set.SporogenesisDegreeDays;
popParams.MinTempThreshold = set.MinTempThresholdCelsius;
popParams.MosquitoDeathTemperature = set.DeathTempCelsius;;
popParams.MosquitoLifespanDays = new TimeSpan((int)set.LifespanDays, 0, 0, 0);;
popParams.SliceLength = new TimeSpan((int)set.SliceLengthHours, 0, 0);
popParams.MaxTempSuitability = set.MaxTSNormaliser;
// max ts for default settings is 34.2467; the Weiss code had 33.89401 , not sure how generated.
// I got this using iterative solver in excel.
Console.WriteLine("Tile data loaded, computation beginning");
var mConfig = _cfg.modelRunConfig;
float[][] tsOut = new float[numCells][];
DateTime[] outputDates = null;
ParallelOptions b = new ParallelOptions();
if (mConfig.MaxThreads != 0)
{
// set threads to 1 for easier step-through debugging or some other number to not hog the whole machine
b.MaxDegreeOfParallelism = (int)mConfig.MaxThreads;
//System.Threading.ThreadPool.SetMaxThreads(set.MaxThreads, set.MaxThreads);
//System.Threading.ThreadPool.SetMinThreads(set.MinThreads, set.MinThreads);
}
int testnum = 0;
while (outputDates == null && testnum < numCells)
{
// if we haven't got at least 50% of the data and 100+ points it's probably crap
// (unless we're playing with synoptic data or something, so make this threshold configurable).
// This doesn't affect the date calculation but the spline will throw an error
// on initialisation
var nValid = Math.Min(
maxTempData[testnum].Count(v => v != _maxReader.NoDataValue),
minTempData[testnum].Count(v => v != _minReader.NoDataValue));
if (nValid < nFiles / 2 || nValid < mConfig.MinRequiredDataPoints)
{
testnum += 1;
continue;
}
// set up a dummy model purely to parse the output dates (that they will all share)
// avoids the need to test for the first one to do this in the parallel loop, which needs a lock,
// which slows things right down with >>20 cores
TSCellModel tsModel = new TSCellModel(
popParams,
new GeographicCellLocation()
{
Latitude = lats[0], Longitude = lons[0]
},
PopulationTypes.Pointers);
if (!tsModel.SetData(maxTempData[testnum], minTempData[testnum], dates, _maxReader.NoDataValue.Value,
mConfig.MaxTempFilesAreLST, mConfig.MinTempFilesAreLST))
{
throw new ApplicationException("Pop goes the weasel");
}
;
outputDates = tsModel.OutputDates;
break;
}
Parallel.For(0, numCells, b, c =>
{
if (lsMask[c] != mConfig.MaskValidValue)
{
// this cell is in the sea, no need to run, return as a special case a zero length result array
tsOut[c] = new float[0];
}
else
{
int rownum = c / xSize;
int colnum = c % xSize;
GeographicCellLocation geogParams = new GeographicCellLocation();
geogParams.Latitude = lats[rownum];
geogParams.Longitude = lons[colnum];
//geogParams.ModelRuntimeDays = nDays;
//geogParams.StartJulianDay = 0;
// run only if we've got at least 50% of the data and 100+ points (or whatever is configured)
var nValid = Math.Min(
maxTempData[c].Count(v => v != _maxReader.NoDataValue),
minTempData[c].Count(v => v != _minReader.NoDataValue));
if (nValid < nFiles / 2 || nValid < mConfig.MinRequiredDataPoints)
{
tsOut[c] = new float[0];
}
else
{
TSCellModel tsModel = new TSCellModel(popParams, geogParams, PopulationTypes.Pointers);
// var dd = dayData[c];
// var nd = nightData[c];
// float optimal = 28.6857194664029F;
// for (int i = 0; i < 727; i++) {
// dd[i] = optimal;
// if (i%3==0) { dd[i] = 43; }
// nd[i] = optimal; }
//tsModel.SetData(dd, nd, dates, _maxReader.NoDataValue.Value, false);
if (!tsModel.SetData(maxTempData[c], minTempData[c], dates, _maxReader.NoDataValue.Value,
mConfig.MaxTempFilesAreLST, mConfig.MinTempFilesAreLST))
{
throw new ApplicationException("Pop goes the weasel");
}
;
// run the entire ts model for this location
float[] tsCell = tsModel.RunModel();
int nOutputPoints = tsCell.Length;
tsOut[c] = tsCell;
/*lock (_lockobj) // ensure only 1 thread makes this check
* {
* // calculate and record the dates of the outputs for an arbitrary one of the cell models
* if (outputDates == null)
* {
* outputDates = tsModel.OutputDates;
* }
* }*/
}
}
}
);
if (_outputDates == null)
{
_outputDates = outputDates;
}
else if (outputDates != null && !_outputDates.SequenceEqual(outputDates))
{
throw new ApplicationException("Dates have changed between tiles somehow, this shouldn't happen...");
}
return(tsOut);
}
/// <summary>
/// Runs TS model for all cells of all tiles required to cover the given bounding box at the given tile size.
/// Any tile wholly in the sea will be skipped (no output files generated)
/// </summary>
/// <param name="WestDegrees"></param>
/// <param name="EastDegrees"></param>
/// <param name="NorthDegrees"></param>
/// <param name="SouthDegrees"></param>
/// <param name="TileSize"></param>
public void RunAllTiles()
{
var globalGT = _maxReader.GeoTransform;
var lims = _cfg.spatialLimits;
var pxOverall = GDAL_Operations.CalculatePixelCoordsOfBlock(globalGT,
lims.WestLimitDegrees, lims.EastLimitDegrees,
lims.NorthLimitDegrees, lims.SouthLimitDegrees
);
var TileSize = _cfg.modelRunConfig.MaxTileSizePx;
var latsToRun = _maxReader.GetSubsetLatitudeCoords((int)pxOverall.NorthPixelCoord, (int)(pxOverall.SouthPixelCoord - pxOverall.NorthPixelCoord));
var lonsToRun = _maxReader.GetSubsetLongitudeCoords((int)pxOverall.WestPixelCoord, (int)(pxOverall.EastPixelCoord - pxOverall.WestPixelCoord));
var nTilesX = (int)Math.Ceiling((double)lonsToRun.Length / TileSize);
var nTilesY = (int)Math.Ceiling((double)latsToRun.Length / TileSize);
var nTilesTotal = nTilesX * nTilesY;
string tileDir = lims.WestLimitDegrees.ToString() + "W-"
+ lims.EastLimitDegrees.ToString() + "E-"
+ lims.NorthLimitDegrees.ToString() + "N-"
+ lims.SouthLimitDegrees.ToString() + "S-"
+ TileSize.ToString() + "px_tiles";
Console.WriteLine("Initiating run of " + nTilesTotal.ToString() + " tiles");
var runDir = nTilesTotal == 1 ?
_cfg.dataPathConfig.OutputFolder :
System.IO.Path.Combine(_cfg.dataPathConfig.OutputFolder, tileDir);
System.IO.Directory.CreateDirectory(runDir);
for (int tileRow = 0; tileRow < nTilesY; tileRow++)
{
int yOff = (int)pxOverall.NorthPixelCoord + tileRow * TileSize;
int yEnd = yOff + TileSize;
int thisTileYSize = TileSize;
if (yEnd > pxOverall.SouthPixelCoord)
{
thisTileYSize = (int)pxOverall.SouthPixelCoord - yOff;
}
for (int tileCol = 0; tileCol < nTilesX; tileCol++)
{
var tileNum = tileRow * nTilesX + tileCol + 1;
string tilenameLocPart;
if (nTilesTotal > 1)
{
tilenameLocPart = ".r" + tileRow.ToString("D3") + "_c" + tileCol.ToString("D3") + ".tif";
}
else
{
tilenameLocPart = ".tif";
}
if (System.IO.Directory.EnumerateFiles(runDir, "*" + tilenameLocPart).Count() != 0)
{
System.Console.WriteLine("Tile " + tileNum.ToString() + " appears to be already done, skipping");
continue;
}
int xOff = (int)pxOverall.WestPixelCoord + tileCol * TileSize;
int xEnd = xOff + TileSize;
int thisTileXSize = TileSize;
if (xEnd > pxOverall.EastPixelCoord)
{
thisTileXSize = (int)pxOverall.EastPixelCoord - xOff;
}
var tileCoords = new PixelLims(xOff, xOff + thisTileXSize, yOff, yOff + thisTileYSize);
var cellRes = RunTile(xOff, yOff, thisTileXSize, thisTileYSize);
if (cellRes.Length == 0)
{
// whole tile was in masked / sea area. Do not bother to write output.
Console.WriteLine("Tile " + tileNum.ToString() + " was wholly in sea - skipped");
continue;
}
Console.WriteLine("Tile computation completed, writing output");
var tileRes = TransposeCellData(
cellRes,
thisTileXSize, thisTileYSize);
var tileGT = _maxReader.GetSubsetGeoTransform(tileCoords);
var tileProj = _maxReader.Projection;
// write each tile to a separate tiff file - we can mosaic them later.
var tileLims = new PixelLims(0, thisTileXSize, 0, thisTileYSize);
for (int t = 0; t < tileRes.Length; t++)
{
var tData = tileRes[t];
string fn = _cfg.modelRunConfig.OutputFileTag + "." +
_outputDates[t].Date.ToString("yyyy.MM") + tilenameLocPart;
string outFile = System.IO.Path.Combine(runDir, fn);
GDAL_Operations.WriteWholeTiff(outFile, tData, tileGT, tileProj, tileLims, true, _maxReader.NoDataValue);
}
Console.WriteLine("Tile " + tileNum.ToString() + " finished - wrote " + tileRes.Length.ToString() + " files");
}
}
}
