/// <summary>
/// Parses ASCII lines file.
/// </summary>
internal static IEnumerable <Chunk> AsciiLines(Func <byte[], int, double, Chunk?> lineParser,
string filename, ParseConfig config
)
{
var fileSizeInBytes = new FileInfo(filename).Length;
var stream = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read);
return(AsciiLines(lineParser, stream, fileSizeInBytes, config));
}
/// <summary></summary> public IEnumerable <Chunk> ParseFile(string filename, ParseConfig config) => f_parseFile(filename, config);
/// <summary></summary> public PointFileInfo ParseFileInfo(string filename, ParseConfig config) => f_parseFileInfo(filename, config);
/// <summary>
/// Parses ASCII lines stream.
/// </summary>
internal static IEnumerable <Chunk> AsciiLines(Func <byte[], int, double, Chunk?> lineParser,
Stream stream, long streamLengthInBytes, ParseConfig config
)
{
// importing file
var result = stream
.ChunkStreamAtNewlines(streamLengthInBytes, config.ReadBufferSizeInBytes, config.CancellationToken)
.ParseBuffers(streamLengthInBytes, lineParser, config.MinDist, config.MaxDegreeOfParallelism, config.Verbose, config.CancellationToken)
;
//var foo = result.ToArray();
return(result);
}
/// <summary></summary>
public ImportConfig WithVerbose(bool x) => new ImportConfig(this)
{
ParseConfig = ParseConfig.WithVerbose(x)
};
/// <summary></summary>
public ImportConfig WithReadBufferSizeInBytes(int x) => new ImportConfig(this)
{
ParseConfig = ParseConfig.WithReadBufferSizeInBytes(x)
};
/// <summary></summary>
public ImportConfig WithMaxChunkPointCount(int x) => new ImportConfig(this)
{
ParseConfig = ParseConfig.WithMaxChunkPointCount(Math.Max(x, 1))
};
/// <summary></summary>
public ImportConfig WithMinDist(double x) => new ImportConfig(this)
{
ParseConfig = ParseConfig.WithMinDist(x)
};
/// <summary></summary>
public ImportConfig WithMaxDegreeOfParallelism(int x) => new ImportConfig(this)
{
ParseConfig = ParseConfig.WithMaxDegreeOfParallelism(x)
};
/// <summary></summary>
public ImportConfig WithCancellationToken(CancellationToken x) => new ImportConfig(this)
{
ParseConfig = ParseConfig.WithCancellationToken(x)
};
/// <summary>
/// Returns chunk with duplicate point positions removed.
/// </summary>
public Chunk ImmutableFilterMinDistByCell(Cell bounds, ParseConfig config)
{
if (!HasPositions)
{
return(this);
}
var smallestCellExponent = Fun.Log2(config.MinDist).Ceiling();
var positions = Positions;
var take = new bool[Count];
var foo = new List <int>(positions.Count); for (var i = 0; i < positions.Count; i++)
{
foo.Add(i);
}
filter(bounds, foo).Wait();
async Task filter(Cell c, List <int> ia)
{
#if DEBUG
if (ia == null || ia.Count == 0)
{
throw new InvalidOperationException();
}
if (c.Exponent < smallestCellExponent)
{
throw new InvalidOperationException();
}
#endif
if (c.Exponent == smallestCellExponent)
{
take[ia[0]] = true;
return;
}
var center = c.GetCenter();
var subias = new List <int> [8].SetByIndex(_ => new List <int>());
for (var i = 0; i < ia.Count; i++)
{
var p = positions[ia[i]];
var o = 0;
if (p.X >= center.X)
{
o = 1;
}
if (p.Y >= center.Y)
{
o |= 2;
}
if (p.Z >= center.Z)
{
o |= 4;
}
subias[o].Add(ia[i]);
}
var ts = new List <Task>();
for (var i = 0; i < 8; i++)
{
if (subias[i].Count == 0)
{
continue;
}
if (subias[i].Count == 1)
{
take[subias[i][0]] = true; continue;
}
var _i = i;
var t = (subias[i].Count < 16384)
? filter(c.GetOctant(i), subias[i])
: Task.Run(() => filter(c.GetOctant(_i), subias[_i]))
;
ts.Add(t);
}
await Task.WhenAll(ts);
}
var self = this;
var ps = Positions.Where((_, i) => take[i]).ToList();
var cs = HasColors ? Colors.Where((_, i) => take[i]).ToList() : null;
var ns = HasNormals ? Normals.Where((_, i) => take[i]).ToList() : null;
var js = HasIntensities ? Intensities.Where((_, i) => take[i]).ToList() : null;
if (config.Verbose)
{
var removedCount = this.Count - ps.Count;
if (removedCount > 0)
{
//Report.Line($"[ImmutableFilterMinDistByCell] {this.Count:N0} - {removedCount:N0} -> {ps.Count:N0}");
}
}
return(new Chunk(ps, cs, ns, js));
}
/// <summary>
/// </summary>
public static IEnumerable <Chunk> ImmutableUnmixOutOfCore(this IEnumerable <Chunk> chunks, string tmpdir, int binsExponent, ParseConfig config)
{
var binsExponentFactor = 1.0 / Math.Pow(2.0, binsExponent);
try
{
Report.BeginTimed("ImmutableUnmixOutOfCore");
tmpdir = Path.Combine(tmpdir, Guid.NewGuid().ToString());
Directory.CreateDirectory(tmpdir);
var root = default(Cell?);
var hasNormals = false;
var hasColors = false;
var hasIntensities = false;
var countChunks = 0L;
var countOriginal = 0L;
Report.BeginTimed("processing chunks");
var lockedFilenames = new HashSet <string>();
Parallel.ForEach(chunks, chunk =>
{
countChunks++;
countOriginal += chunk.Count;
hasNormals = chunk.HasNormals;
hasColors = chunk.HasColors;
hasIntensities = chunk.HasIntensities;
var _ps = chunk.Positions;
var _ns = chunk.Normals;
var _js = chunk.Intensities;
var _cs = chunk.Colors;
// binning
var map = new Dictionary <V3l, List <int> >();
for (var i = 0; i < chunk.Count; i++)
{
var p = _ps[i];
var key = binsExponent == 0 ? new V3l(p) : new V3l(p * binsExponentFactor);
if (!map.TryGetValue(key, out var value))
{
map[key] = value = new List <int>();
}
value.Add(i);
}
// store cells
foreach (var kv in map)
{
var cell = new Cell(kv.Key.X, kv.Key.Y, kv.Key.Z, binsExponent);
root = root.HasValue ? new Cell(new Box3d(root.Value.BoundingBox, cell.BoundingBox)) : cell;
var filename = Path.Combine(tmpdir, $"{kv.Key.X}_{kv.Key.Y}_{kv.Key.Z}");
while (true)
{
lock (lockedFilenames)
{
if (lockedFilenames.Add(filename))
{
break;
}
}
Task.Delay(100);
}
using (var f = File.Open(filename, FileMode.Append, FileAccess.Write, FileShare.None))
using (var bw = new BinaryWriter(f))
{
var ia = kv.Value;
foreach (var i in ia)
{
var p = _ps[i];
bw.Write(p.X); bw.Write(p.Y); bw.Write(p.Z);
if (hasNormals)
{
var n = _ns[i]; bw.Write(n.X); bw.Write(n.Y); bw.Write(n.Z);
}
if (hasIntensities)
{
var j = _js[i]; bw.Write(j);
}
if (hasColors)
{
var c = _cs[i]; var x = c.R + c.G << 8 + c.B << 16; bw.Write(x);
}
}
}
lock (lockedFilenames)
{
lockedFilenames.Remove(filename);
}
}
});
Report.EndTimed();
Report.Line($"[ImmutableUnmixOutOfCore] chunk count = {countChunks:N0}");
Report.Line($"[ImmutableUnmixOutOfCore] root cell = {root:N0}");
Report.Line($"[ImmutableUnmixOutOfCore] point count = {countOriginal:N0}");
// construct hierarchy
Report.BeginTimed("constructing hierarchy");
foreach (var path in Directory.EnumerateFiles(tmpdir))
{
var filename = Path.GetFileName(path);
var ts = filename.Split('_');
var cell = new Cell(long.Parse(ts[0]), long.Parse(ts[1]), long.Parse(ts[2]), binsExponent);
var stack = new Stack <string>();
while (cell.Exponent < root.Value.Exponent)
{
cell = cell.Parent;
stack.Push($"{cell.X}_{cell.Y}_{cell.Z}_{cell.Exponent}");
}
var dir = tmpdir;
while (stack.Count > 0)
{
dir = Path.Combine(dir, stack.Pop());
}
try
{
Directory.CreateDirectory(dir);
File.Move(path, Path.Combine(dir, filename));
}
catch (Exception e)
{
Report.Error(e.ToString());
Report.Error($"[dir ] {dir}");
Report.Error($"[move] {path} -> {Path.Combine(dir, filename)}");
}
}
Report.EndTimed();
// filter min distance
Report.BeginTimed("filtering min distance");
var countFiltered = 0L;
Parallel.ForEach(Directory.EnumerateFiles(tmpdir, "*", SearchOption.AllDirectories), path =>
{
var filename = Path.GetFileName(path);
var ts = filename.Split('_');
var cell = new Cell(long.Parse(ts[0]), long.Parse(ts[1]), long.Parse(ts[2]), binsExponent);
var _ps = new List <V3d>();
var _ns = hasNormals ? new List <V3f>() : null;
var _js = hasIntensities ? new List <int>() : null;
var _cs = hasColors ? new List <C4b>() : null;
using (var f = File.Open(path, FileMode.Open, FileAccess.Read))
using (var br = new BinaryReader(f))
{
try
{
while (br.BaseStream.Position < br.BaseStream.Length)
{
_ps.Add(new V3d(br.ReadDouble(), br.ReadDouble(), br.ReadDouble()));
if (hasNormals)
{
_ns.Add(new V3f(br.ReadSingle(), br.ReadSingle(), br.ReadSingle()));
}
if (hasIntensities)
{
_js.Add(br.ReadInt32());
}
if (hasColors)
{
var x = br.ReadInt32(); _cs.Add(new C4b(x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff));
}
}
}
catch (Exception e)
{
Report.Error(e.ToString());
return;
}
}
var chunk = new Chunk(_ps, _cs, _ns, _js);
var chunkFiltered = chunk.ImmutableFilterMinDistByCell(cell, config);
countFiltered += chunkFiltered.Count;
//Report.Line($"[{cell}] {countFiltered:N0}/{countOriginal:N0} ({countOriginal- countFiltered:N0})");
using (var f = File.Open(path, FileMode.Open, FileAccess.Write, FileShare.None))
using (var bw = new BinaryWriter(f))
{
for (var i = 0; i < chunkFiltered.Count; i++)
{
var p = chunkFiltered.Positions[i];
bw.Write(p.X); bw.Write(p.Y); bw.Write(p.Z);
if (hasNormals)
{
var n = chunkFiltered.Normals[i]; bw.Write(n.X); bw.Write(n.Y); bw.Write(n.Z);
}
if (hasIntensities)
{
var j = chunkFiltered.Intensities[i]; bw.Write(j);
}
if (hasColors)
{
var c = chunkFiltered.Colors[i]; var x = c.R + c.G << 8 + c.B << 16; bw.Write(x);
}
}
}
});
Report.Line($"{countFiltered:N0}/{countOriginal:N0} (removed {countOriginal - countFiltered:N0} points)");
Report.EndTimed();
// return final chunks
var ps = new List <V3d>();
var ns = hasNormals ? new List <V3f>() : null;
var js = hasIntensities ? new List <int>() : null;
var cs = hasColors ? new List <C4b>() : null;
foreach (var path in Directory.EnumerateFiles(tmpdir, "*", SearchOption.AllDirectories))
{
using (var f = File.Open(path, FileMode.Open, FileAccess.Read))
using (var br = new BinaryReader(f))
{
try
{
while (br.BaseStream.Position < br.BaseStream.Length)
{
ps.Add(new V3d(br.ReadDouble(), br.ReadDouble(), br.ReadDouble()));
if (hasNormals)
{
ns.Add(new V3f(br.ReadSingle(), br.ReadSingle(), br.ReadSingle()));
}
if (hasIntensities)
{
js.Add(br.ReadInt32());
}
if (hasColors)
{
var x = br.ReadInt32(); cs.Add(new C4b(x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff));
}
}
}
catch (Exception e)
{
Report.Error(e.ToString());
ps = new List <V3d>();
ns = hasNormals ? new List <V3f>() : null;
js = hasIntensities ? new List <int>() : null;
cs = hasColors ? new List <C4b>() : null;
continue;
}
}
File.Delete(path);
if (ps.Count >= config.MaxChunkPointCount)
{
yield return(new Chunk(ps, cs, ns, js));
ps = new List <V3d>();
ns = hasNormals ? new List <V3f>() : null;
js = hasIntensities ? new List <int>() : null;
cs = hasColors ? new List <C4b>() : null;
}
}
// rest?
if (ps.Count >= 0)
{
yield return(new Chunk(ps, cs, ns, js));
}
}
finally
{
try
{
Report.BeginTimed("deleting temporary data");
Directory.Delete(tmpdir, true);
Report.EndTimed();
}
catch (Exception e)
{
Report.Warn(e.ToString());
}
Report.EndTimed();
}
}
