public void UploadLocalLayer(string workspaceId, string id)
{
if (HasLayer(workspaceId, id))
{
_tileStorage.Store($"{workspaceId}/{id}.mbtiles", _tileCache.GetFilePath(workspaceId, id));
}
}
protected override async Task Process()
{
// there's two types of conversion to consider.
// 1. spatial source data arrives and is placed in storage, we get a message and convert it
// to geojson using gdal, and put the result in storage. We add a new req to the queue to
// convert the geojson to mbtile.
// 2. the geojson from the previous step (or possibly geojson directly) is in storage, we get
// a message and convert to mbtile and place result in storage.
QueuedConversionJob queued = null;
try
{
queued = await _mbConversionQueue.GetJob();
}
catch (Exception ex)
{
_logger.LogError($"MapBox Conversion failed to retrieve queued job", ex);
}
if (queued != null) // if no job queued, don't try
{
using (var workFolder = new TemporaryWorkFolder())
{
try
{
var job = queued.Content;
if (job?.DataLocation != null && job?.LayerId != null && job?.WorkspaceId != null)
// if the job has missing values, don't process it, just delete it from queue.
{
// convert the geoJSON to a mapbox dataset
var timer = new Stopwatch();
timer.Start();
// retrieve the geoJSON file from the supplied URI
var inputFilePath = await new Uri(job.DataLocation).DownloadToLocal(workFolder.Path);
var mbTilesFilePath = Path.Combine(workFolder.Path, $"{job.LayerId}.mbtiles");
var processArgument = GetProcessArgument(job.LayerName, job.Description, mbTilesFilePath, inputFilePath);
_logger.LogDebug($"executing tippecanoe process with argument {processArgument}");
var executionResult =
ProcessExecutionService.ExecuteProcess(ConverterFileName, processArgument);
if (executionResult.success)
{
_logger.LogDebug($"tippecanoe process successfully executed");
}
else
{
_logger.LogError($"tippecanoe process failed: {executionResult.error}");
}
// now we need to put the converted mbtile file into storage
await _tileStorage.Store($"{job.WorkspaceId}/{job.LayerId}.mbtiles", mbTilesFilePath);
_logger.LogDebug("Upload of mbtile file to storage complete.");
timer.Stop();
_logger.LogDebug($"MapBox Conversion finished for Layer {job.LayerId} in {timer.ElapsedMilliseconds} ms.");
try
{
await _statusTable.UpdateStatus(job.WorkspaceId, job.LayerId, LayerStatus.Finished);
_logger.LogDebug($"Layer {job.LayerId} status updated to Finished");
}
catch (Exception ex)
{
_logger.LogError($"Error when updating Layer {job.LayerId} status to Finished", ex);
throw;
}
await _topicClient.SendMessage(new MapLayerUpdateData
{
MapLayerId = job.LayerId,
WorkspaceId = job.WorkspaceId,
Type = MapLayerUpdateType.Update
});
}
await _mbConversionQueue.DeleteJob(queued);
_logger.LogDebug("Deleted MapBox Conversion message");
}
catch (Exception ex)
{
if (queued.DequeueCount >= RetryLimit)
{
try
{
await _mbConversionQueue.DeleteJob(queued);
if (queued.Content?.LayerId != null && queued.Content?.WorkspaceId != null)
{
await _statusTable.UpdateStatus(queued.Content.WorkspaceId, queued.Content.LayerId, LayerStatus.Failed);
}
_logger.LogError($"MapBox Conversion failed for layer {queued.Content?.LayerId} after reaching retry limit", ex);
}
catch (Exception e)
{
_logger.LogError($"MapBox Conversion failed to clear bad conversion for layer {queued.Content?.LayerId}", e);
}
}
else
{
_logger.LogWarning($"MapBox Conversion failed for layer {queued.Content?.LayerId} and will retry later", ex);
}
}
}
}
else
{
await Task.Delay(_serviceOptions.ConvertPolling);
}
}