public unsafe void ComputeExtent(ProgressManager progressManager) { using (var process = progressManager.StartProcess("CalculateLASExtent")) { short pointSizeBytes = PointSizeBytes; int minX = 0, minY = 0, minZ = 0; int maxX = 0, maxY = 0, maxZ = 0; foreach (var chunk in GetBlockEnumerator(process)) { if (minX == 0 && maxX == 0) { var p = (SQuantizedPoint3D*)chunk.PointDataPtr; minX = maxX = (*p).X; minY = maxY = (*p).Y; minZ = maxZ = (*p).Z; } byte* pb = chunk.PointDataPtr; while (pb < chunk.PointDataEndPtr) { var p = (SQuantizedPoint3D*)pb; if ((*p).X < minX) minX = (*p).X; else if ((*p).X > maxX) maxX = (*p).X; if ((*p).Y < minY) minY = (*p).Y; else if ((*p).Y > maxY) maxY = (*p).Y; if ((*p).Z < minZ) minZ = (*p).Z; else if ((*p).Z > maxZ) maxZ = (*p).Z; pb += pointSizeBytes; } if (!process.Update(chunk)) break; } var quantizedExtent = new SQuantizedExtent3D(minX, minY, minZ, maxX, maxY, maxZ); m_extent = m_header.Quantization.Convert(quantizedExtent); process.LogTime("Traversed {0:0,0} points", Count); } }
public unsafe PointCloudBinarySource ConvertTextToBinary(string binaryPath, ProgressManager progressManager) { short pointSizeBytes = 3 * sizeof(double); double minX = 0, minY = 0, minZ = 0; double maxX = 0, maxY = 0, maxZ = 0; int pointCount = 0; using (var process = progressManager.StartProcess("ConvertTextToBinary")) { BufferInstance inputBuffer = process.AcquireBuffer(true); BufferInstance outputBuffer = process.AcquireBuffer(true); int pointsPerBuffer = outputBuffer.Length / pointSizeBytes; int usableBytesPerBuffer = pointsPerBuffer * pointSizeBytes; byte* inputBufferPtr = inputBuffer.DataPtr; byte* outputBufferPtr = outputBuffer.DataPtr; int bufferIndex = 0; int skipped = 0; using (var inputStream = StreamManager.OpenReadStream(FilePath)) { long inputLength = inputStream.Length; long estimatedOutputLength = inputLength; using (var outputStream = StreamManager.OpenWriteStream(binaryPath, estimatedOutputLength, 0, true)) { int bytesRead; int readStart = 0; while ((bytesRead = inputStream.Read(inputBuffer.Data, readStart, inputBuffer.Length - readStart)) > 0) { bytesRead += readStart; readStart = 0; int i = 0; while (i < bytesRead) { // identify line start while (i < bytesRead && (inputBufferPtr[i] == '\r' || inputBufferPtr[i] == '\n')) { ++i; } int lineStart = i; // identify line end while (i < bytesRead && inputBufferPtr[i] != '\r' && inputBufferPtr[i] != '\n') { ++i; } // handle buffer overlap if (i == bytesRead) { Array.Copy(inputBuffer.Data, lineStart, inputBuffer.Data, 0, i - lineStart); readStart = i - lineStart; break; } // this may get overwritten if this is not a valid parse double* p = (double*)(outputBufferPtr + bufferIndex); if (!ParseXYZFromLine(inputBufferPtr, lineStart, i, p)) { ++skipped; continue; } if (pointCount == 0) { minX = maxX = p[0]; minY = maxY = p[1]; minZ = maxZ = p[2]; } else { if (p[0] < minX) minX = p[0]; else if (p[0] > maxX) maxX = p[0]; if (p[1] < minY) minY = p[1]; else if (p[1] > maxY) maxY = p[1]; if (p[2] < minZ) minZ = p[2]; else if (p[2] > maxZ) maxZ = p[2]; } bufferIndex += pointSizeBytes; ++pointCount; // write usable buffer chunk if (usableBytesPerBuffer == bufferIndex) { outputStream.Write(outputBuffer.Data, 0, bufferIndex); bufferIndex = 0; } } if (!process.Update((float)inputStream.Position / inputLength)) break; } // write remaining buffer if (bufferIndex > 0) outputStream.Write(outputBuffer.Data, 0, bufferIndex); } } process.Log("Skipped {0} lines", skipped); process.LogTime("Copied {0:0,0} points", pointCount); } var extent = new Extent3D(minX, minY, minZ, maxX, maxY, maxZ); var source = new PointCloudBinarySource(this, pointCount, extent, null, 0, pointSizeBytes); return source; }
public PointCloudTileSource TilePointFileIndex(LASFile tiledFile, BufferInstance segmentBuffer, ProgressManager progressManager) { var stopwatch = new Stopwatch(); stopwatch.Start(); var analysis = AnalyzePointFile(segmentBuffer.Length, progressManager); var quantizedExtent = m_source.QuantizedExtent; var tileCounts = analysis.Density.GetTileCountsForInitialization(); var fileSize = tiledFile.PointDataOffset + (m_source.PointSizeBytes * m_source.Count); AttemptFastAllocate(tiledFile.FilePath, fileSize); var lowResPointCountMax = PROPERTY_MAX_LOWRES_POINTS.Value; var lowResBuffer = BufferManager.AcquireBuffer(m_id, lowResPointCountMax * m_source.PointSizeBytes); var lowResWrapper = new PointBufferWrapper(lowResBuffer, m_source.PointSizeBytes, lowResPointCountMax); var validTiles = analysis.GridIndex.Sum(r => r.GridRange.ValidCells); var lowResPointsPerTile = lowResPointCountMax / validTiles; var lowResTileSize = (ushort)Math.Sqrt(lowResPointsPerTile); var lowResGrid = Grid<int>.Create(lowResTileSize, lowResTileSize, true, -1); var lowResCounts = tileCounts.Copy<int>(); using (var outputStream = StreamManager.OpenWriteStream(tiledFile.FilePath, fileSize, tiledFile.PointDataOffset)) { var i = 0; foreach (var segment in analysis.GridIndex) { progressManager.Log("~ Processing Index Segment {0}/{1}", ++i, analysis.GridIndex.Count); var sparseSegment = m_source.CreateSparseSegment(segment); var sparseSegmentWrapper = new PointBufferWrapper(segmentBuffer, sparseSegment); var tileRegionFilter = new TileRegionFilter(tileCounts, quantizedExtent, segment.GridRange); // this call will fill the buffer with points, add the counts, and sort QuantTilePointsIndexed(sparseSegment, sparseSegmentWrapper, tileRegionFilter, tileCounts, lowResWrapper, lowResGrid, lowResCounts, progressManager); var segmentFilteredPointCount = tileRegionFilter.GetCellOrdering().Sum(t => tileCounts.Data[t.Row, t.Col]); var segmentFilteredBytes = segmentFilteredPointCount * sparseSegmentWrapper.PointSizeBytes; // write out the buffer using (var process = progressManager.StartProcess("WriteIndexSegment")) { var segmentBufferIndex = 0; foreach (var tile in segment.GridRange.GetCellOrdering()) { var tileCount = tileCounts.Data[tile.Row, tile.Col]; if (tileCount > 0) { var tileSize = (tileCount - lowResCounts.Data[tile.Row, tile.Col]) * sparseSegmentWrapper.PointSizeBytes; outputStream.Write(sparseSegmentWrapper.Data, segmentBufferIndex, tileSize); segmentBufferIndex += tileSize; if (!process.Update((float)segmentBufferIndex / segmentFilteredBytes)) break; } } } if (progressManager.IsCanceled()) break; } // write low-res var lowResActualPointCount = lowResCounts.Data.Cast<int>().Sum(); outputStream.Write(lowResWrapper.Data, 0, lowResActualPointCount * lowResWrapper.PointSizeBytes); } var actualDensity = new PointCloudTileDensity(tileCounts, m_source.Quantization); var tileSet = new PointCloudTileSet(m_source, actualDensity, tileCounts, lowResCounts); var tileSource = new PointCloudTileSource(tiledFile, tileSet, analysis.Statistics); if (!progressManager.IsCanceled()) tileSource.IsDirty = false; tileSource.WriteHeader(); return tileSource; }
private static unsafe void QuantTilePointsIndexed(IPointCloudBinarySource source, PointBufferWrapper segmentBuffer, TileRegionFilter tileFilter, SQuantizedExtentGrid<int> tileCounts, PointBufferWrapper lowResBuffer, Grid<int> lowResGrid, Grid<int> lowResCounts, ProgressManager progressManager) { var quantizedExtent = source.QuantizedExtent; // generate counts and add points to buffer using (var process = progressManager.StartProcess("QuantTilePointsIndexedFilter")) { var group = new ChunkProcessSet(tileFilter, segmentBuffer); group.Process(source.GetBlockEnumerator(process)); } // sort points in buffer using (var process = progressManager.StartProcess("QuantTilePointsIndexedSort")) { var tilePositions = tileFilter.CreatePositionGrid(segmentBuffer, source.PointSizeBytes); var sortedCount = 0; foreach (var tile in tileFilter.GetCellOrdering()) { var currentPosition = tilePositions[tile.Row, tile.Col]; if (currentPosition.IsIncomplete) { while (currentPosition.IsIncomplete) { var p = (SQuantizedPoint3D*)currentPosition.DataPtr; var targetPosition = tilePositions[ (((*p).Y - quantizedExtent.MinY) / tileCounts.CellSizeY), (((*p).X - quantizedExtent.MinX) / tileCounts.CellSizeX) ]; if (targetPosition.DataPtr != currentPosition.DataPtr) { // the point tile is not the current traversal tile, // so swap the points and resume on the swapped point targetPosition.Swap(currentPosition.DataPtr); } else { // this point is in the correct tile, move on currentPosition.Increment(); } ++sortedCount; } if (!process.Update((float)sortedCount / segmentBuffer.PointCount)) break; } } } // TEST /*using (var process = progressManager.StartProcess("QuantTilePointsIndexedTEST")) { //var templateQuantizedExtent = quantizedExtent.ComputeQuantizedTileExtent(new SimpleGridCoord(0, 0), tileCounts); //var cellSizeX = (int)(templateQuantizedExtent.RangeX / lowResGrid.SizeX); //var cellSizeY = (int)(templateQuantizedExtent.RangeY / lowResGrid.SizeY); var sX2 = Math.Pow(source.Quantization.ScaleFactorX, 2); var sY2 = Math.Pow(source.Quantization.ScaleFactorY, 2); var sZ2 = Math.Pow(source.Quantization.ScaleFactorZ, 2); var index = 0; foreach (var tile in tileFilter.GetCellOrdering()) { var count = tileCounts.Data[tile.Row, tile.Col]; var dataPtr = segmentBuffer.PointDataPtr + (index * source.PointSizeBytes); var dataEndPtr = dataPtr + (count * source.PointSizeBytes); //var tileQuantizedExtent = quantizedExtent.ComputeQuantizedTileExtent(tile, tileCounts); //lowResGrid.Reset(); //var grid = Grid<int>.Create(); var pb = dataPtr; while (pb < dataEndPtr) { var p = (SQuantizedPoint3D*)pb; // meters? var searchRadius = 1; var pointsWithinRadius = 0; var pb2 = dataPtr; //while (pb2 < dataEndPtr) while (pb2 < (byte*)Math.Min((long)dataEndPtr, (long)(dataPtr + 20 * source.PointSizeBytes))) { var p2 = (SQuantizedPoint3D*)pb2; //var d2 = // sX2 * Math.Pow((*p2).X - (*p).X, 2) + // sY2 * Math.Pow((*p2).Y - (*p).Y, 2) + // sZ2 * Math.Pow((*p2).Z - (*p).Z, 2); //if (Math.Sqrt(d2) < searchRadius) //{ // ++pointsWithinRadius; //} pb2 += source.PointSizeBytes; } //(*p).Z = (int)(quantizedExtent.MinX + (pointsWithinRadius * 100) / source.Quantization.ScaleFactorZ); //var cellX = (((*p).X - tileQuantizedExtent.MinX) / cellSizeX); //var cellY = (((*p).Y - tileQuantizedExtent.MinY) / cellSizeY); //var offset = lowResGrid.Data[cellY, cellX]; //if (offset == -1) //{ // lowResGrid.Data[cellY, cellX] = (int)(pb - segmentBuffer.PointDataPtr); //} //else //{ // var pBest = (SQuantizedPoint3D*)(segmentBuffer.PointDataPtr + offset); // if ((*p).Z < (*pBest).Z) // lowResGrid.Data[cellY, cellX] = (int)(pb - segmentBuffer.PointDataPtr); //} pb += source.PointSizeBytes; ++index; } if (!process.Update((float)index / segmentBuffer.PointCount)) break; } }*/ // determine representative low-res points for each tile and swap them to a new buffer using (var process = progressManager.StartProcess("QuantTilePointsIndexedExtractLowRes")) { var removedBytes = 0; var templateQuantizedExtent = quantizedExtent.ComputeQuantizedTileExtent(new SimpleGridCoord(0, 0), tileCounts); var cellSizeX = (int)(templateQuantizedExtent.RangeX / lowResGrid.SizeX); var cellSizeY = (int)(templateQuantizedExtent.RangeY / lowResGrid.SizeY); var index = 0; foreach (var tile in tileFilter.GetCellOrdering()) { var count = tileCounts.Data[tile.Row, tile.Col]; var dataPtr = segmentBuffer.PointDataPtr + (index * source.PointSizeBytes); var dataEndPtr = dataPtr + (count * source.PointSizeBytes); var tileQuantizedExtent = quantizedExtent.ComputeQuantizedTileExtent(tile, tileCounts); lowResGrid.Reset(); var pb = dataPtr; while (pb < dataEndPtr) { var p = (SQuantizedPoint3D*)pb; var cellX = (((*p).X - tileQuantizedExtent.MinX) / cellSizeX); var cellY = (((*p).Y - tileQuantizedExtent.MinY) / cellSizeY); // todo: make lowResGrid <long> to avoid cast? var offset = lowResGrid.Data[cellY, cellX]; if (offset == -1) { lowResGrid.Data[cellY, cellX] = (int)(pb - segmentBuffer.PointDataPtr); } else { var pBest = (SQuantizedPoint3D*)(segmentBuffer.PointDataPtr + offset); //if ((*p).Z > (*pBest).Z) if ((*p).Z < (*pBest).Z) lowResGrid.Data[cellY, cellX] = (int)(pb - segmentBuffer.PointDataPtr); //var cellCenterX = (cellX + 0.5) * cellSizeX; //var cellCenterY = (cellY + 0.5) * cellSizeY; //var bd2 = DistanceRatioFromPointToCellCenter2(pBest, cellCenterX, cellCenterY, cellSizeX, cellSizeY); //var cd2 = DistanceRatioFromPointToCellCenter2(p, cellCenterX, cellCenterY, cellSizeX, cellSizeY); //if (cd2 < bd2) // lowResGrid.Data[cellY, cellX] = (int)(pb - segmentBuffer.PointDataPtr); } pb += source.PointSizeBytes; ++index; } // ignore boundary points lowResGrid.ClearOverflow(); // sort valid cells var offsets = lowResGrid.Data.Cast<int>().Where(v => v != lowResGrid.FillVal).ToArray(); if (offsets.Length > 0) { Array.Sort(offsets); // shift the data up to the first offset var tileSrc = (int)(dataPtr - segmentBuffer.PointDataPtr); Buffer.BlockCopy(segmentBuffer.Data, tileSrc, segmentBuffer.Data, (tileSrc - removedBytes), (offsets[0] - tileSrc)); // pack the remaining points for (var i = 0; i < offsets.Length; i++) { var currentOffset = offsets[i]; // copy point to buffer lowResBuffer.Append(segmentBuffer.Data, currentOffset, source.PointSizeBytes); // shift everything between here and the next offset var copyEnd = (i == offsets.Length - 1) ? (dataEndPtr - segmentBuffer.PointDataPtr) : (offsets[i + 1]); var copySrc = (currentOffset + source.PointSizeBytes); var copyDst = (currentOffset - removedBytes); var copyLen = (int)(copyEnd - copySrc); Buffer.BlockCopy(segmentBuffer.Data, copySrc, segmentBuffer.Data, copyDst, copyLen); removedBytes += source.PointSizeBytes; } lowResCounts.Data[tile.Row, tile.Col] = offsets.Length; } if (!process.Update((float)index / segmentBuffer.PointCount)) break; } } }
private static PointCloudAnalysisResult QuantEstimateDensity(IPointCloudBinarySource source, int maxSegmentLength, SQuantizedExtentGrid<int> tileCounts, ProgressManager progressManager) { Statistics stats = null; List<PointCloudBinarySourceEnumeratorSparseGridRegion> gridIndexSegments = null; var extent = source.Extent; var quantizedExtent = source.QuantizedExtent; var statsMapping = new ScaledStatisticsMapping(quantizedExtent.MinZ, quantizedExtent.RangeZ, 1024); var gridCounter = new GridCounter(source, tileCounts); using (var process = progressManager.StartProcess("QuantEstimateDensity")) { var group = new ChunkProcessSet(gridCounter, statsMapping); group.Process(source.GetBlockEnumerator(process)); } stats = statsMapping.ComputeStatistics(extent.MinZ, extent.RangeZ); var density = new PointCloudTileDensity(tileCounts, source.Quantization); gridIndexSegments = gridCounter.GetGridIndex(density, maxSegmentLength); var result = new PointCloudAnalysisResult(density, stats, source.Quantization, gridIndexSegments); return result; }
private void GeneratePreviewPixelGrid(ushort maxPreviewDimension, ProgressManager progressManager) { var gridSet = new GridQuantizedSet(this, maxPreviewDimension); var group = new ChunkProcessSet(gridSet); using (var process = progressManager.StartProcess("GeneratePreviewPixelGrid")) { group.Process(GetTileEnumerator(process)); } m_pixelGridSet = gridSet; }
public PointCloudTileSource Process(ProgressManager progressManager) { progressManager.Log("<= {0}", m_inputHandler.FilePath); PerformanceManager.Start(m_inputHandler.FilePath); // check for existing tile source LoadFromCache(progressManager); if (m_tileSource == null) { using (var process = progressManager.StartProcess("ProcessSet")) { m_binarySource = m_inputHandler.GenerateBinarySource(progressManager); m_tiledHandler = LASFile.Create(m_tiledHandler.FilePath, m_binarySource); using (var segmentBuffer = BufferManager.AcquireBuffer(m_id, (int)PROPERTY_SEGMENT_SIZE.Value, true)) { var tileManager = new PointCloudTileManager(m_binarySource); m_tileSource = tileManager.TilePointFileIndex(m_tiledHandler, segmentBuffer, progressManager); } #warning this was for xyz, but I have not yet re-implemented that anyway //if (m_binarySource.FilePath != m_inputHandler.FilePath) // File.Delete(m_binarySource.FilePath); if (m_tileSource.IsDirty) { m_tileSource.Close(); File.Delete(m_tileSource.FilePath); m_tileSource = null; process.LogTime("=> Processing Cancelled"); } else { process.LogTime("=> Processing Completed"); } } GC.Collect(); } TransferRate averageReadSpeed = PerformanceManager.GetReadSpeed(); TransferRate averageWriteSpeed = PerformanceManager.GetWriteSpeed(); Context.WriteLine("IO Read Speed: {0}", averageReadSpeed); Context.WriteLine("IO Write Speed: {0}", averageWriteSpeed); //{ // // test // Stopwatch stopwatch = new Stopwatch(); // stopwatch.Start(); // PointCloudTile tempTile = m_tileSource.TileSet[0, 0]; // Grid<float> grid = new Grid<float>(tempTile.Extent, 540, (float)m_tileSource.Extent.MinZ - 1.0f, true); // Grid<uint> quantizedGrid = new Grid<uint>(grid.SizeX, grid.SizeY, m_tileSource.Extent, true); // using (GridTileSource<float> gridSource = new GridTileSource<float>(m_tiledPath + ".grid", grid.SizeX, grid.SizeY, m_tileSource.TileSet.Cols, m_tileSource.TileSet.Rows)) // { // int tempBufferSize = (int)(m_tileSource.TileSet.Max(t => t.PointCount)); // byte[] tempBuffer = new byte[tempBufferSize * m_tileSource.PointSizeBytes]; // foreach (PointCloudTile tile in m_tileSource) // { // m_tileSource.LoadTileGrid(tile, tempBuffer, grid, quantizedGrid); // gridSource.WriteTile(tile.Col, tile.Row, grid.Data); // if (!progressManager.Update((float)tile.Index / m_tileSource.TileSet.TileCount)) // break; // } // //gridSource.ReadTile(tempTile.Col, tempTile.Row, grid.Data); // } // m_tileSource.Close(); // progressManager.Log(stopwatch, "Generated GRID"); //} return m_tileSource; }