protected override async Task Process()
{
// _logger.LogDebug("GDALConversion process starting.");
// For the enlightenment of other, later, readers:
// ogr2ogr will be used to process not only obvious conversion sources (eg shape files) but also
// geojson files. Why, you might ask, because tippecanoe can import GeoJSON directly? It's because
// passing the GeoJSON through ogr2ogr will ensure that the final GeoJSON is in the correct projection
// and that it should be valid GeoJSON as well.
var gdalMsg = await _mapdata.GdConversionQueue.GetMessage();
if (gdalMsg != null) // if no message, don't try
{
try
{
ConversionMessageData gdalData;
try
{
gdalData = JsonConvert.DeserializeObject <ConversionMessageData>(gdalMsg.Content);
}
catch (JsonReaderException e)
{
_logger.LogError($"failed to decode JSON message on queue {e}");
return;
}
var start = DateTime.UtcNow;
_logger.LogDebug("About to convert Gdal");
var randomFolderName = Path.GetRandomFileName();
_logger.LogDebug($"folder name = {randomFolderName}");
// it will be in the system's temporary directory
var tempPath = Path.Combine(Path.GetTempPath(), randomFolderName);
_logger.LogDebug($"temporary path = {tempPath}");
// If the directory already existed, throw an error - this should never happen, but just in case.
if (Directory.Exists(tempPath))
{
_logger.LogError($"The temporary path '{tempPath}' already existed.");
throw new Exception("The temporary path already existed.");
}
// Try to create the directory.
Directory.CreateDirectory(tempPath);
_logger.LogDebug($"The directory was created successfully at {Directory.GetCreationTime(tempPath)}.");
// we need to keep source and dest separate in case there's a collision in filenames.
var sourcePath = Path.Combine(tempPath, "source");
Directory.CreateDirectory(sourcePath);
var destPath = Path.Combine(tempPath, "dest");
Directory.CreateDirectory(destPath);
if (gdalData.DataLocation != null) // if it was null we don't want to do anything except remove the job from queue
{
// retrieve the source data file from the supplied URI
var remoteUri = new Uri(gdalData.DataLocation);
// we will need to know if this is a zip file later so that we can tell GDAL to use the zip virtual file system.
var isZip = Path.GetExtension(remoteUri.AbsolutePath).ToUpper() == ".ZIP";
var localFile = Path.Combine(sourcePath, Path.GetFileName(remoteUri.AbsolutePath));
WebClient myWebClient = new WebClient();
_logger.LogDebug($"Downloading {gdalData.DataLocation} to {localFile}");
myWebClient.DownloadFile(gdalData.DataLocation, localFile);
List <string> filesToProcess = new List <string>(); // a list of files to convert
// There could be multiple files if we were given a zip file - currently we look for shp and gdb.
if (isZip)
{
// if it was a zip file we will extract it and use the content as the input.
using (var zip = ZipFile.OpenRead(localFile))
{
zip.ExtractToDirectory(sourcePath);
}
// Shape files are handled differently because there are usually multiple files per shape and we don't want to process the supporting files.
filesToProcess.AddRange(Directory.GetFiles(sourcePath, "*.shp"));
// GDB is also processed differently because they are actually directories, not files, but are treated by GDAL like they were files.
filesToProcess.AddRange(Directory.GetDirectories(sourcePath, "*.gdb"));
// for the moment we'll add geojson too. There's bound to be others we'll want to support but this should get us going.
filesToProcess.AddRange(Directory.GetFiles(sourcePath, "*.geojson"));
}
else
{
// not a zip file so we will assume that it's a single file to convert.
filesToProcess.Add(localFile);
}
foreach (var sourceFile in filesToProcess)
{
var layerName = Path.GetFileNameWithoutExtension(sourceFile);
var geoJsonFile = Path.Combine(destPath, $"{layerName}.geojson");
var gdalProcess = new Process
{
StartInfo =
{
FileName = "ogr2ogr",
Arguments =
"-f \"GeoJSON\" " + // always converting to GeoJSON
$"-nln \"{layerName}\" " +
"-t_srs \"EPSG:4326\" " + // always transform to WGS84
$"{geoJsonFile} " +
$"{sourceFile}",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
}
};
_logger.LogDebug($"ogr2ogr arguments are {gdalProcess.StartInfo.Arguments}");
gdalProcess.Start();
var errmsg = "";
while (!gdalProcess.StandardError.EndOfStream)
{
errmsg += gdalProcess.StandardError.ReadLine();
}
gdalProcess.WaitForExit();
var exitCode = gdalProcess.ExitCode;
_logger.LogDebug($"og2ogr returned exit code {exitCode}");
if (exitCode != 0)
{
_logger.LogError($"Spatial data to GeoJSON conversion failed (errcode={exitCode}), msgs = {errmsg}");
throw new Exception($"Spatial data to GeoJSON conversion failed. {errmsg}");
}
_logger.LogDebug($"geojson file is in {geoJsonFile}");
// now we need to put the converted geojson file into storage
var location = await _mapdata.GeojsonContainer.Store($"{layerName}.geojson", geoJsonFile);
_logger.LogDebug("Upload of geojson file to storage complete.");
var end = DateTime.UtcNow;
var duration = end - start;
_logger.LogDebug($"GDALConversion took {duration.TotalMilliseconds} ms.");
// we created geoJson so we can put a request in for geojson to mvt conversion.
var mbRequestId = await _mapdata.CreateMbConversionRequest(new ConversionMessageData
{
DataLocation = location,
Description = gdalData.Description,
LayerName = layerName
});
await _mapdata.JobStatusTable.AddStatus(mbRequestId, gdalMsg.Id);
}
}
// we completed GDAL conversion and creation of MVT conversion request, so remove the GDAL request from the queue
_logger.LogDebug("deleting gdal message from queue");
await _mapdata.GdConversionQueue.DeleteMessage(gdalMsg);
await _mapdata.JobStatusTable.UpdateStatus(gdalMsg.Id, JobStatus.Finished);
}
catch (Exception)
{
await _mapdata.JobStatusTable.UpdateStatus(gdalMsg.Id, JobStatus.Failed);
throw;
}
}
await Task.Delay(_options.CheckConvertTime);
}
