public async Task <ActionResult> Invoke([FromBody] CreateLayerDto dto) { var id = Guid.NewGuid().ToString(); ConversionJobData job = new ConversionJobData { LayerId = id, WorkspaceId = WorkspaceId, LayerName = dto.Name, DataLocation = dto.DataLocation, Description = dto.Description }; await _gdConversionQueue.Queue(job); await _statusTable.UpdateStatus(WorkspaceId, id, LayerStatus.Processing); if (dto.Styles != null) { await _layerStyleService.Update(WorkspaceId, id, dto.Styles); } return(Accepted(new LayerStatusDto { Id = id, Status = LayerStatus.Processing })); }
public async Task <ActionResult> Invoke([FromBody] UpdateLayerDto dto) { await _validator.Validate(dto); var status = await _statusTable.GetStatus(WorkspaceId, LayerId); if (!status.HasValue) { return(NotFound()); } if (dto.DataLocation != null) { // in other cases we will have to create a new conversion job var job = new ConversionJobData { LayerId = LayerId, WorkspaceId = WorkspaceId, LayerName = dto.Name, DataLocation = dto.DataLocation, Description = dto.Description }; await _gdConversionQueue.Queue(job); await _statusTable.UpdateStatus(WorkspaceId, LayerId, LayerStatus.Processing); } return(Accepted(new LayerStatusDto { Id = LayerId, Status = LayerStatus.Processing })); }
protected override async Task Process() { // For the enlightenment of other, later, readers: // ogr2ogr will be used to process not only obvious conversion sources (eg shape files) but also // geojson files. Why, you might ask, because tippecanoe can import GeoJSON directly? It's because // passing the GeoJSON through ogr2ogr will ensure that the final GeoJSON is in the correct projection // and that it should be valid GeoJSON as well. QueuedConversionJob queued = null; try { queued = await _gdConversionQueue.GetJob(); } catch (Exception ex) { _logger.LogError($"GdalConversion failed to retrieve queued job", ex); } if (queued != null) // if no job queued, don't try { using (var workFolder = new TemporaryWorkFolder()) { try { var job = queued.Content; if (job?.DataLocation != null && job.LayerId != null && job.WorkspaceId != null) // if the job has missing values, don't process it, just delete it from queue. { var timer = new Stopwatch(); timer.Start(); _logger.LogDebug($"Processing GDAL Conversion for Layer {queued.Content.LayerId} within Queue Message {queued.Id}"); // Keep source and dest separate in case of file name collision. var sourcePath = workFolder.CreateSubFolder("source"); var destPath = workFolder.CreateSubFolder("dest"); var downloadedFilePath = await new Uri(job.DataLocation).DownloadToLocal(sourcePath); var inputFilePath = GetGdalInputFileParameter(downloadedFilePath, workFolder); var geoJsonFile = Path.Combine(destPath, $"{job.LayerId}.geojson"); var processArgument = GetProcessArgument(job.LayerName, geoJsonFile, inputFilePath); _logger.LogDebug($"executing ogr2ogr process with argument {processArgument}"); var executionResult = ProcessExecutionService.ExecuteProcess(ConverterFileName, processArgument); if (executionResult.success) { _logger.LogDebug($"ogr2ogr process successfully executed"); } else { _logger.LogError($"ogr2ogr process failed: {executionResult.error}"); throw new Exception($"ogr2ogr process failed: {executionResult.error}"); } // now we need to put the converted geojson file into storage var location = await _geoJsonStorage.Store($"{job.WorkspaceId}/{job.LayerId}.geojson", geoJsonFile); _logger.LogDebug("Upload of geojson file to storage complete."); timer.Stop(); _logger.LogDebug($"GDAL Conversion finished for Layer {job.LayerId} in {timer.ElapsedMilliseconds} ms."); // we created geoJson so we can put a request in for geojson to mvt conversion. await _mbConversionQueue.Queue(new ConversionJobData { LayerId = job.LayerId, WorkspaceId = job.WorkspaceId, LayerName = job.LayerName, Description = job.Description, DataLocation = location }); } // we completed GDAL conversion and creation of MVT conversion request, so remove the GDAL request from the queue await _gdConversionQueue.DeleteJob(queued); _logger.LogDebug("GDAL Conversion message deleted "); } catch (Exception ex) { if (queued.DequeueCount >= RetryLimit) { try { await _gdConversionQueue.DeleteJob(queued); if (queued.Content?.LayerId != null && queued.Content?.WorkspaceId != null) { await _statusTable.UpdateStatus(queued.Content.WorkspaceId, queued.Content.LayerId, LayerStatus.Failed); } _logger.LogError($"GDAL Conversion failed for layer {queued.Content?.LayerId} after reaching retry limit", ex); } catch (Exception e) { _logger.LogError($"GDAL Conversion failed to clear bad conversion for layer {queued.Content?.LayerId}", e); } } else { _logger.LogWarning($"GDAL Conversion failed for layer {queued.Content?.LayerId} will retry later", ex); } } } } else { await Task.Delay(_serviceOptions.ConvertPolling); } }
protected override async Task Process() { // there's two types of conversion to consider. // 1. spatial source data arrives and is placed in storage, we get a message and convert it // to geojson using gdal, and put the result in storage. We add a new req to the queue to // convert the geojson to mbtile. // 2. the geojson from the previous step (or possibly geojson directly) is in storage, we get // a message and convert to mbtile and place result in storage. QueuedConversionJob queued = null; try { queued = await _mbConversionQueue.GetJob(); } catch (Exception ex) { _logger.LogError($"MapBox Conversion failed to retrieve queued job", ex); } if (queued != null) // if no job queued, don't try { using (var workFolder = new TemporaryWorkFolder()) { try { var job = queued.Content; if (job?.DataLocation != null && job?.LayerId != null && job?.WorkspaceId != null) // if the job has missing values, don't process it, just delete it from queue. { // convert the geoJSON to a mapbox dataset var timer = new Stopwatch(); timer.Start(); // retrieve the geoJSON file from the supplied URI var inputFilePath = await new Uri(job.DataLocation).DownloadToLocal(workFolder.Path); var mbTilesFilePath = Path.Combine(workFolder.Path, $"{job.LayerId}.mbtiles"); var processArgument = GetProcessArgument(job.LayerName, job.Description, mbTilesFilePath, inputFilePath); _logger.LogDebug($"executing tippecanoe process with argument {processArgument}"); var executionResult = ProcessExecutionService.ExecuteProcess(ConverterFileName, processArgument); if (executionResult.success) { _logger.LogDebug($"tippecanoe process successfully executed"); } else { _logger.LogError($"tippecanoe process failed: {executionResult.error}"); } // now we need to put the converted mbtile file into storage await _tileStorage.Store($"{job.WorkspaceId}/{job.LayerId}.mbtiles", mbTilesFilePath); _logger.LogDebug("Upload of mbtile file to storage complete."); timer.Stop(); _logger.LogDebug($"MapBox Conversion finished for Layer {job.LayerId} in {timer.ElapsedMilliseconds} ms."); try { await _statusTable.UpdateStatus(job.WorkspaceId, job.LayerId, LayerStatus.Finished); _logger.LogDebug($"Layer {job.LayerId} status updated to Finished"); } catch (Exception ex) { _logger.LogError($"Error when updating Layer {job.LayerId} status to Finished", ex); throw; } await _topicClient.SendMessage(new MapLayerUpdateData { MapLayerId = job.LayerId, WorkspaceId = job.WorkspaceId, Type = MapLayerUpdateType.Update }); } await _mbConversionQueue.DeleteJob(queued); _logger.LogDebug("Deleted MapBox Conversion message"); } catch (Exception ex) { if (queued.DequeueCount >= RetryLimit) { try { await _mbConversionQueue.DeleteJob(queued); if (queued.Content?.LayerId != null && queued.Content?.WorkspaceId != null) { await _statusTable.UpdateStatus(queued.Content.WorkspaceId, queued.Content.LayerId, LayerStatus.Failed); } _logger.LogError($"MapBox Conversion failed for layer {queued.Content?.LayerId} after reaching retry limit", ex); } catch (Exception e) { _logger.LogError($"MapBox Conversion failed to clear bad conversion for layer {queued.Content?.LayerId}", e); } } else { _logger.LogWarning($"MapBox Conversion failed for layer {queued.Content?.LayerId} and will retry later", ex); } } } } else { await Task.Delay(_serviceOptions.ConvertPolling); } }