public Task <QueuedConversionJob> GetJob()
{
System.Diagnostics.Debug.WriteLine($"get message from {_queueDatabase.DataSource} (Enter)");
CheckExpiredMessages();
QueuedConversionJob queued = null;
long popCount1;
// now we can look for an entry to process.
using (var getMsgCmd = new SqliteCommand())
{
getMsgCmd.Connection = _queueDatabase;
getMsgCmd.CommandText = "SELECT ID, PopCount, Content from Queue WHERE PopReceipt IS NULL LIMIT 1";
var reader = getMsgCmd.ExecuteReader();
var popReceipt = Guid.NewGuid().ToString();
var pop = reader["PopCount"];
if (DBNull.Value.Equals(pop))
{
popCount1 = 0;
}
else
{
popCount1 = (long)pop;
}
popCount1++;
try
{
queued = new QueuedConversionJob
{
Id = reader["ID"].ToString(),
PopReceipt = popReceipt,
Content = JsonConvert.DeserializeObject <ConversionJobData>(reader["Content"].ToString())
};
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine($"Exception thrown while getting new queued job - {ex}");
return(null);
}
}
using (var updateMsgCmd = new SqliteCommand())
{
updateMsgCmd.Connection = _queueDatabase;
updateMsgCmd.CommandText =
"UPDATE Queue SET PopReceipt = $popReceipt, PopCount = $popCount, Popped = datetime('now') WHERE ID=$ID";
updateMsgCmd.Parameters.AddWithValue("$ID", queued.Id);
updateMsgCmd.Parameters.AddWithValue("$popReceipt", queued.PopReceipt);
updateMsgCmd.Parameters.AddWithValue("$popCount", popCount1);
updateMsgCmd.ExecuteReader();
}
System.Diagnostics.Debug.WriteLine($"get message from {_queueDatabase.DataSource} (Exit)");
return(string.IsNullOrWhiteSpace(queued.Id)
? Task.FromResult <QueuedConversionJob>(null)
: Task.FromResult(queued));
}
public Task DeleteJob(QueuedConversionJob job)
{
System.Diagnostics.Debug.WriteLine($"deleting message from queue {_queueDatabase.DataSource} (Enter)");
using (var delMsgCmd = new SqliteCommand())
{
delMsgCmd.Connection = _queueDatabase;
delMsgCmd.CommandText = "DELETE FROM Queue WHERE ID=$ID AND PopReceipt=$popReceipt";
delMsgCmd.Parameters.AddWithValue("$ID", job.Id);
delMsgCmd.Parameters.AddWithValue("$popReceipt", job.PopReceipt);
delMsgCmd.ExecuteReader();
}
System.Diagnostics.Debug.WriteLine($"deleting message from queue {_queueDatabase.DataSource} (Exit)");
return(Task.CompletedTask);
}
public async Task DeleteJob(QueuedConversionJob job)
{
await _queue.DeleteMessageAsync(job.Id, job.PopReceipt);
}
protected override async Task Process()
{
// For the enlightenment of other, later, readers:
// ogr2ogr will be used to process not only obvious conversion sources (eg shape files) but also
// geojson files. Why, you might ask, because tippecanoe can import GeoJSON directly? It's because
// passing the GeoJSON through ogr2ogr will ensure that the final GeoJSON is in the correct projection
// and that it should be valid GeoJSON as well.
QueuedConversionJob queued = null;
try
{
queued = await _gdConversionQueue.GetJob();
}
catch (Exception ex)
{
_logger.LogError($"GdalConversion failed to retrieve queued job", ex);
}
if (queued != null) // if no job queued, don't try
{
using (var workFolder = new TemporaryWorkFolder())
{
try
{
var job = queued.Content;
if (job?.DataLocation != null && job.LayerId != null && job.WorkspaceId != null)
// if the job has missing values, don't process it, just delete it from queue.
{
var timer = new Stopwatch();
timer.Start();
_logger.LogDebug($"Processing GDAL Conversion for Layer {queued.Content.LayerId} within Queue Message {queued.Id}");
// Keep source and dest separate in case of file name collision.
var sourcePath = workFolder.CreateSubFolder("source");
var destPath = workFolder.CreateSubFolder("dest");
var downloadedFilePath = await new Uri(job.DataLocation).DownloadToLocal(sourcePath);
var inputFilePath = GetGdalInputFileParameter(downloadedFilePath, workFolder);
var geoJsonFile = Path.Combine(destPath, $"{job.LayerId}.geojson");
var processArgument = GetProcessArgument(job.LayerName, geoJsonFile, inputFilePath);
_logger.LogDebug($"executing ogr2ogr process with argument {processArgument}");
var executionResult =
ProcessExecutionService.ExecuteProcess(ConverterFileName, processArgument);
if (executionResult.success)
{
_logger.LogDebug($"ogr2ogr process successfully executed");
}
else
{
_logger.LogError($"ogr2ogr process failed: {executionResult.error}");
throw new Exception($"ogr2ogr process failed: {executionResult.error}");
}
// now we need to put the converted geojson file into storage
var location = await _geoJsonStorage.Store($"{job.WorkspaceId}/{job.LayerId}.geojson", geoJsonFile);
_logger.LogDebug("Upload of geojson file to storage complete.");
timer.Stop();
_logger.LogDebug($"GDAL Conversion finished for Layer {job.LayerId} in {timer.ElapsedMilliseconds} ms.");
// we created geoJson so we can put a request in for geojson to mvt conversion.
await _mbConversionQueue.Queue(new ConversionJobData
{
LayerId = job.LayerId,
WorkspaceId = job.WorkspaceId,
LayerName = job.LayerName,
Description = job.Description,
DataLocation = location
});
}
// we completed GDAL conversion and creation of MVT conversion request, so remove the GDAL request from the queue
await _gdConversionQueue.DeleteJob(queued);
_logger.LogDebug("GDAL Conversion message deleted ");
}
catch (Exception ex)
{
if (queued.DequeueCount >= RetryLimit)
{
try
{
await _gdConversionQueue.DeleteJob(queued);
if (queued.Content?.LayerId != null && queued.Content?.WorkspaceId != null)
{
await _statusTable.UpdateStatus(queued.Content.WorkspaceId, queued.Content.LayerId,
LayerStatus.Failed);
}
_logger.LogError($"GDAL Conversion failed for layer {queued.Content?.LayerId} after reaching retry limit", ex);
}
catch (Exception e)
{
_logger.LogError($"GDAL Conversion failed to clear bad conversion for layer {queued.Content?.LayerId}", e);
}
}
else
{
_logger.LogWarning($"GDAL Conversion failed for layer {queued.Content?.LayerId} will retry later", ex);
}
}
}
}
else
{
await Task.Delay(_serviceOptions.ConvertPolling);
}
}
protected override async Task Process()
{
// there's two types of conversion to consider.
// 1. spatial source data arrives and is placed in storage, we get a message and convert it
// to geojson using gdal, and put the result in storage. We add a new req to the queue to
// convert the geojson to mbtile.
// 2. the geojson from the previous step (or possibly geojson directly) is in storage, we get
// a message and convert to mbtile and place result in storage.
QueuedConversionJob queued = null;
try
{
queued = await _mbConversionQueue.GetJob();
}
catch (Exception ex)
{
_logger.LogError($"MapBox Conversion failed to retrieve queued job", ex);
}
if (queued != null) // if no job queued, don't try
{
using (var workFolder = new TemporaryWorkFolder())
{
try
{
var job = queued.Content;
if (job?.DataLocation != null && job?.LayerId != null && job?.WorkspaceId != null)
// if the job has missing values, don't process it, just delete it from queue.
{
// convert the geoJSON to a mapbox dataset
var timer = new Stopwatch();
timer.Start();
// retrieve the geoJSON file from the supplied URI
var inputFilePath = await new Uri(job.DataLocation).DownloadToLocal(workFolder.Path);
var mbTilesFilePath = Path.Combine(workFolder.Path, $"{job.LayerId}.mbtiles");
var processArgument = GetProcessArgument(job.LayerName, job.Description, mbTilesFilePath, inputFilePath);
_logger.LogDebug($"executing tippecanoe process with argument {processArgument}");
var executionResult =
ProcessExecutionService.ExecuteProcess(ConverterFileName, processArgument);
if (executionResult.success)
{
_logger.LogDebug($"tippecanoe process successfully executed");
}
else
{
_logger.LogError($"tippecanoe process failed: {executionResult.error}");
}
// now we need to put the converted mbtile file into storage
await _tileStorage.Store($"{job.WorkspaceId}/{job.LayerId}.mbtiles", mbTilesFilePath);
_logger.LogDebug("Upload of mbtile file to storage complete.");
timer.Stop();
_logger.LogDebug($"MapBox Conversion finished for Layer {job.LayerId} in {timer.ElapsedMilliseconds} ms.");
try
{
await _statusTable.UpdateStatus(job.WorkspaceId, job.LayerId, LayerStatus.Finished);
_logger.LogDebug($"Layer {job.LayerId} status updated to Finished");
}
catch (Exception ex)
{
_logger.LogError($"Error when updating Layer {job.LayerId} status to Finished", ex);
throw;
}
await _topicClient.SendMessage(new MapLayerUpdateData
{
MapLayerId = job.LayerId,
WorkspaceId = job.WorkspaceId,
Type = MapLayerUpdateType.Update
});
}
await _mbConversionQueue.DeleteJob(queued);
_logger.LogDebug("Deleted MapBox Conversion message");
}
catch (Exception ex)
{
if (queued.DequeueCount >= RetryLimit)
{
try
{
await _mbConversionQueue.DeleteJob(queued);
if (queued.Content?.LayerId != null && queued.Content?.WorkspaceId != null)
{
await _statusTable.UpdateStatus(queued.Content.WorkspaceId, queued.Content.LayerId, LayerStatus.Failed);
}
_logger.LogError($"MapBox Conversion failed for layer {queued.Content?.LayerId} after reaching retry limit", ex);
}
catch (Exception e)
{
_logger.LogError($"MapBox Conversion failed to clear bad conversion for layer {queued.Content?.LayerId}", e);
}
}
else
{
_logger.LogWarning($"MapBox Conversion failed for layer {queued.Content?.LayerId} and will retry later", ex);
}
}
}
}
else
{
await Task.Delay(_serviceOptions.ConvertPolling);
}
}
