/// <summary> /// Parses ASCII lines file. /// </summary> internal static IEnumerable <Chunk> AsciiLines(Func <byte[], int, double, Chunk?> lineParser, string filename, ParseConfig config ) { var fileSizeInBytes = new FileInfo(filename).Length; var stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read); return(AsciiLines(lineParser, stream, fileSizeInBytes, config)); }
/// <summary></summary> public IEnumerable <Chunk> ParseFile(string filename, ParseConfig config) => f_parseFile(filename, config);
/// <summary></summary> public PointFileInfo ParseFileInfo(string filename, ParseConfig config) => f_parseFileInfo(filename, config);
/// <summary> /// Parses ASCII lines stream. /// </summary> internal static IEnumerable <Chunk> AsciiLines(Func <byte[], int, double, Chunk?> lineParser, Stream stream, long streamLengthInBytes, ParseConfig config ) { // importing file var result = stream .ChunkStreamAtNewlines(streamLengthInBytes, config.ReadBufferSizeInBytes, config.CancellationToken) .ParseBuffers(streamLengthInBytes, lineParser, config.MinDist, config.MaxDegreeOfParallelism, config.Verbose, config.CancellationToken) ; //var foo = result.ToArray(); return(result); }
/// <summary></summary> public ImportConfig WithVerbose(bool x) => new ImportConfig(this) { ParseConfig = ParseConfig.WithVerbose(x) };
/// <summary></summary> public ImportConfig WithReadBufferSizeInBytes(int x) => new ImportConfig(this) { ParseConfig = ParseConfig.WithReadBufferSizeInBytes(x) };
/// <summary></summary> public ImportConfig WithMaxChunkPointCount(int x) => new ImportConfig(this) { ParseConfig = ParseConfig.WithMaxChunkPointCount(Math.Max(x, 1)) };
/// <summary></summary> public ImportConfig WithMinDist(double x) => new ImportConfig(this) { ParseConfig = ParseConfig.WithMinDist(x) };
/// <summary></summary> public ImportConfig WithMaxDegreeOfParallelism(int x) => new ImportConfig(this) { ParseConfig = ParseConfig.WithMaxDegreeOfParallelism(x) };
/// <summary></summary> public ImportConfig WithCancellationToken(CancellationToken x) => new ImportConfig(this) { ParseConfig = ParseConfig.WithCancellationToken(x) };
/// <summary> /// Returns chunk with duplicate point positions removed. /// </summary> public Chunk ImmutableFilterMinDistByCell(Cell bounds, ParseConfig config) { if (!HasPositions) { return(this); } var smallestCellExponent = Fun.Log2(config.MinDist).Ceiling(); var positions = Positions; var take = new bool[Count]; var foo = new List <int>(positions.Count); for (var i = 0; i < positions.Count; i++) { foo.Add(i); } filter(bounds, foo).Wait(); async Task filter(Cell c, List <int> ia) { #if DEBUG if (ia == null || ia.Count == 0) { throw new InvalidOperationException(); } if (c.Exponent < smallestCellExponent) { throw new InvalidOperationException(); } #endif if (c.Exponent == smallestCellExponent) { take[ia[0]] = true; return; } var center = c.GetCenter(); var subias = new List <int> [8].SetByIndex(_ => new List <int>()); for (var i = 0; i < ia.Count; i++) { var p = positions[ia[i]]; var o = 0; if (p.X >= center.X) { o = 1; } if (p.Y >= center.Y) { o |= 2; } if (p.Z >= center.Z) { o |= 4; } subias[o].Add(ia[i]); } var ts = new List <Task>(); for (var i = 0; i < 8; i++) { if (subias[i].Count == 0) { continue; } if (subias[i].Count == 1) { take[subias[i][0]] = true; continue; } var _i = i; var t = (subias[i].Count < 16384) ? filter(c.GetOctant(i), subias[i]) : Task.Run(() => filter(c.GetOctant(_i), subias[_i])) ; ts.Add(t); } await Task.WhenAll(ts); } var self = this; var ps = Positions.Where((_, i) => take[i]).ToList(); var cs = HasColors ? Colors.Where((_, i) => take[i]).ToList() : null; var ns = HasNormals ? Normals.Where((_, i) => take[i]).ToList() : null; var js = HasIntensities ? Intensities.Where((_, i) => take[i]).ToList() : null; if (config.Verbose) { var removedCount = this.Count - ps.Count; if (removedCount > 0) { //Report.Line($"[ImmutableFilterMinDistByCell] {this.Count:N0} - {removedCount:N0} -> {ps.Count:N0}"); } } return(new Chunk(ps, cs, ns, js)); }
/// <summary> /// </summary> public static IEnumerable <Chunk> ImmutableUnmixOutOfCore(this IEnumerable <Chunk> chunks, string tmpdir, int binsExponent, ParseConfig config) { var binsExponentFactor = 1.0 / Math.Pow(2.0, binsExponent); try { Report.BeginTimed("ImmutableUnmixOutOfCore"); tmpdir = Path.Combine(tmpdir, Guid.NewGuid().ToString()); Directory.CreateDirectory(tmpdir); var root = default(Cell?); var hasNormals = false; var hasColors = false; var hasIntensities = false; var countChunks = 0L; var countOriginal = 0L; Report.BeginTimed("processing chunks"); var lockedFilenames = new HashSet <string>(); Parallel.ForEach(chunks, chunk => { countChunks++; countOriginal += chunk.Count; hasNormals = chunk.HasNormals; hasColors = chunk.HasColors; hasIntensities = chunk.HasIntensities; var _ps = chunk.Positions; var _ns = chunk.Normals; var _js = chunk.Intensities; var _cs = chunk.Colors; // binning var map = new Dictionary <V3l, List <int> >(); for (var i = 0; i < chunk.Count; i++) { var p = _ps[i]; var key = binsExponent == 0 ? new V3l(p) : new V3l(p * binsExponentFactor); if (!map.TryGetValue(key, out var value)) { map[key] = value = new List <int>(); } value.Add(i); } // store cells foreach (var kv in map) { var cell = new Cell(kv.Key.X, kv.Key.Y, kv.Key.Z, binsExponent); root = root.HasValue ? new Cell(new Box3d(root.Value.BoundingBox, cell.BoundingBox)) : cell; var filename = Path.Combine(tmpdir, $"{kv.Key.X}_{kv.Key.Y}_{kv.Key.Z}"); while (true) { lock (lockedFilenames) { if (lockedFilenames.Add(filename)) { break; } } Task.Delay(100); } using (var f = File.Open(filename, FileMode.Append, FileAccess.Write, FileShare.None)) using (var bw = new BinaryWriter(f)) { var ia = kv.Value; foreach (var i in ia) { var p = _ps[i]; bw.Write(p.X); bw.Write(p.Y); bw.Write(p.Z); if (hasNormals) { var n = _ns[i]; bw.Write(n.X); bw.Write(n.Y); bw.Write(n.Z); } if (hasIntensities) { var j = _js[i]; bw.Write(j); } if (hasColors) { var c = _cs[i]; var x = c.R + c.G << 8 + c.B << 16; bw.Write(x); } } } lock (lockedFilenames) { lockedFilenames.Remove(filename); } } }); Report.EndTimed(); Report.Line($"[ImmutableUnmixOutOfCore] chunk count = {countChunks:N0}"); Report.Line($"[ImmutableUnmixOutOfCore] root cell = {root:N0}"); Report.Line($"[ImmutableUnmixOutOfCore] point count = {countOriginal:N0}"); // construct hierarchy Report.BeginTimed("constructing hierarchy"); foreach (var path in Directory.EnumerateFiles(tmpdir)) { var filename = Path.GetFileName(path); var ts = filename.Split('_'); var cell = new Cell(long.Parse(ts[0]), long.Parse(ts[1]), long.Parse(ts[2]), binsExponent); var stack = new Stack <string>(); while (cell.Exponent < root.Value.Exponent) { cell = cell.Parent; stack.Push($"{cell.X}_{cell.Y}_{cell.Z}_{cell.Exponent}"); } var dir = tmpdir; while (stack.Count > 0) { dir = Path.Combine(dir, stack.Pop()); } try { Directory.CreateDirectory(dir); File.Move(path, Path.Combine(dir, filename)); } catch (Exception e) { Report.Error(e.ToString()); Report.Error($"[dir ] {dir}"); Report.Error($"[move] {path} -> {Path.Combine(dir, filename)}"); } } Report.EndTimed(); // filter min distance Report.BeginTimed("filtering min distance"); var countFiltered = 0L; Parallel.ForEach(Directory.EnumerateFiles(tmpdir, "*", SearchOption.AllDirectories), path => { var filename = Path.GetFileName(path); var ts = filename.Split('_'); var cell = new Cell(long.Parse(ts[0]), long.Parse(ts[1]), long.Parse(ts[2]), binsExponent); var _ps = new List <V3d>(); var _ns = hasNormals ? new List <V3f>() : null; var _js = hasIntensities ? new List <int>() : null; var _cs = hasColors ? new List <C4b>() : null; using (var f = File.Open(path, FileMode.Open, FileAccess.Read)) using (var br = new BinaryReader(f)) { try { while (br.BaseStream.Position < br.BaseStream.Length) { _ps.Add(new V3d(br.ReadDouble(), br.ReadDouble(), br.ReadDouble())); if (hasNormals) { _ns.Add(new V3f(br.ReadSingle(), br.ReadSingle(), br.ReadSingle())); } if (hasIntensities) { _js.Add(br.ReadInt32()); } if (hasColors) { var x = br.ReadInt32(); _cs.Add(new C4b(x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff)); } } } catch (Exception e) { Report.Error(e.ToString()); return; } } var chunk = new Chunk(_ps, _cs, _ns, _js); var chunkFiltered = chunk.ImmutableFilterMinDistByCell(cell, config); countFiltered += chunkFiltered.Count; //Report.Line($"[{cell}] {countFiltered:N0}/{countOriginal:N0} ({countOriginal- countFiltered:N0})"); using (var f = File.Open(path, FileMode.Open, FileAccess.Write, FileShare.None)) using (var bw = new BinaryWriter(f)) { for (var i = 0; i < chunkFiltered.Count; i++) { var p = chunkFiltered.Positions[i]; bw.Write(p.X); bw.Write(p.Y); bw.Write(p.Z); if (hasNormals) { var n = chunkFiltered.Normals[i]; bw.Write(n.X); bw.Write(n.Y); bw.Write(n.Z); } if (hasIntensities) { var j = chunkFiltered.Intensities[i]; bw.Write(j); } if (hasColors) { var c = chunkFiltered.Colors[i]; var x = c.R + c.G << 8 + c.B << 16; bw.Write(x); } } } }); Report.Line($"{countFiltered:N0}/{countOriginal:N0} (removed {countOriginal - countFiltered:N0} points)"); Report.EndTimed(); // return final chunks var ps = new List <V3d>(); var ns = hasNormals ? new List <V3f>() : null; var js = hasIntensities ? new List <int>() : null; var cs = hasColors ? new List <C4b>() : null; foreach (var path in Directory.EnumerateFiles(tmpdir, "*", SearchOption.AllDirectories)) { using (var f = File.Open(path, FileMode.Open, FileAccess.Read)) using (var br = new BinaryReader(f)) { try { while (br.BaseStream.Position < br.BaseStream.Length) { ps.Add(new V3d(br.ReadDouble(), br.ReadDouble(), br.ReadDouble())); if (hasNormals) { ns.Add(new V3f(br.ReadSingle(), br.ReadSingle(), br.ReadSingle())); } if (hasIntensities) { js.Add(br.ReadInt32()); } if (hasColors) { var x = br.ReadInt32(); cs.Add(new C4b(x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff)); } } } catch (Exception e) { Report.Error(e.ToString()); ps = new List <V3d>(); ns = hasNormals ? new List <V3f>() : null; js = hasIntensities ? new List <int>() : null; cs = hasColors ? new List <C4b>() : null; continue; } } File.Delete(path); if (ps.Count >= config.MaxChunkPointCount) { yield return(new Chunk(ps, cs, ns, js)); ps = new List <V3d>(); ns = hasNormals ? new List <V3f>() : null; js = hasIntensities ? new List <int>() : null; cs = hasColors ? new List <C4b>() : null; } } // rest? if (ps.Count >= 0) { yield return(new Chunk(ps, cs, ns, js)); } } finally { try { Report.BeginTimed("deleting temporary data"); Directory.Delete(tmpdir, true); Report.EndTimed(); } catch (Exception e) { Report.Warn(e.ToString()); } Report.EndTimed(); } }