private void TestCreateAndReadTimeInstant(TemporalFilter.ValueReferenceType valueRef, TemporalFilter.OperatorType oper)
{
// This general test method tests the serialisation and deserialisation of
// a filter that has a time instant operand
// Creating a time instant
var time = ParseDateTimeInUtc("2018-01-18T03:23:34Z");
var timeInstant = new Item_TimeInstant(time);
var testObject = new TemporalFilter(valueRef, oper, timeInstant);
// Serialise and read
var testObjectIn = SerialiseAndRead(testObject);
// Asserting value reference
Assert.AreEqual(valueRef, testObjectIn.ValueReference);
// Asserting operator
Assert.AreEqual(oper, testObjectIn.Operator);
// Expecting a time instant
var timeInstantIn = (Item_TimeInstant)testObjectIn.Time;
AssertDateTime(time, timeInstantIn.Value);
}
private void SetupFilters(out DecimationFilter decimate, out SpatialFilter spatial, out TemporalFilter temp, out HoleFillingFilter holeFill, out ThresholdFilter threshold)
{
// Colorizer is used to visualize depth data
colorizer = new Colorizer();
// Decimation filter reduces the amount of data (while preserving best samples)
decimate = new DecimationFilter();
decimate.Options[Option.FilterMagnitude].Value = 1.0F;//change scall
// Define spatial filter (edge-preserving)
spatial = new SpatialFilter();
// Enable hole-filling
// Hole filling is an agressive heuristic and it gets the depth wrong many times
// However, this demo is not built to handle holes
// (the shortest-path will always prefer to "cut" through the holes since they have zero 3D distance)
spatial.Options[Option.HolesFill].Value = 1.0F; //change resolution on the edge of image
spatial.Options[Option.FilterMagnitude].Value = 5.0F;
spatial.Options[Option.FilterSmoothAlpha].Value = 1.0F;
spatial.Options[Option.FilterSmoothDelta].Value = 50.0F;
// Define temporal filter
temp = new TemporalFilter();
// Define holefill filter
holeFill = new HoleFillingFilter();
// Aline color to depth
//align_to = new Align(Stream.Depth);
//try to define depth max
threshold = new ThresholdFilter();
threshold.Options[Option.MinDistance].Value = 0;
threshold.Options[Option.MaxDistance].Value = 1;
}
protected override void OnStopStreaming()
{
if (aligner != null)
{
aligner.Dispose();
aligner = null;
}
if (decimationFilter != null)
{
decimationFilter.Dispose();
decimationFilter = null;
}
if (spatialFilter != null)
{
spatialFilter.Dispose();
spatialFilter = null;
}
if (temporalFilter != null)
{
temporalFilter.Dispose();
temporalFilter = null;
}
}
public void TempF_330_CreateAndReadXml_during()
{
// Testing the "during" operator
// Creating a time range
var start = ParseDateTimeInUtc("2018-04-18T03:23:11Z");
var end = ParseDateTimeInUtc("2018-05-18T03:23:11Z");
var timeRange = new Item_TimeRange(start, end);
var testObject = new TemporalFilter(TemporalFilter.ValueReferenceType.PhenomenonTime, TemporalFilter.OperatorType.During, timeRange);
// Serialise and read
var testObjectIn = SerialiseAndRead(testObject);
// Asserting value reference
Assert.AreEqual(TemporalFilter.ValueReferenceType.PhenomenonTime, testObjectIn.ValueReference);
// Asserting operator
Assert.AreEqual(TemporalFilter.OperatorType.During, testObjectIn.Operator);
// Expecting a time range
var timeRangeIn = (Item_TimeRange)testObjectIn.Time;
AssertDateTime(start, timeRangeIn.Start);
AssertDateTime(end, timeRangeIn.End);
}
public void TempF_230_ReadDuring()
{
// Testing the "during" operator
var filepath = TestCommon.TestHelper.TestFileFolder + @"\TemporalFilter_during.xml";
// Getting the proxy
var xmlBytes = System.IO.File.ReadAllBytes(filepath);
var proxy = DeserialiseAndGetProxy(xmlBytes);
// Creating an object from the proxy
var testObject = new TemporalFilter(proxy);
// Asserting value reference
Assert.AreEqual(TemporalFilter.ValueReferenceType.PhenomenonTime, testObject.ValueReference);
// Asserting operator
Assert.AreEqual(TemporalFilter.OperatorType.During, testObject.Operator);
// Expecting a time range
var timeRange = (Item_TimeRange)testObject.Time;
AssertDateTime(ParseDateTimeInUtc("2018-05-18T03:23:11Z"), timeRange.Start);
AssertDateTime(ParseDateTimeInUtc("2018-05-18T03:23:29Z"), timeRange.End);
}
void OnDisable()
{
if (_pb != null)
{
_pb.Dispose();
_pb = null;
}
}
public void Init()
{
_pb = new TemporalFilter();
filterAlphaOpt = _pb.Options[Option.FilterSmoothAlpha];
filterDeltaOpt = _pb.Options[Option.FilterSmoothDelta];
holesFillOpt = _pb.Options[Option.HolesFill];
}
public override ProcessingBlock GetFilter()
{
var filter = new TemporalFilter();
filter.Options[Option.FilterSmoothAlpha].Value = FilterSmoothAlpha;
filter.Options[Option.FilterSmoothDelta].Value = FilterSmoothDelta;
return(filter);
}
private void InitializePostProcessingFilter()
{
decimationFilter = new DecimationFilter();
spatialFilter = new SpatialFilter();
temporalFilter = new TemporalFilter();
// Set some reasonable defaults for now
spatialFilter.Options[Option.HolesFill].Value = 1;
spatialFilter.Options[Option.FilterMagnitude].Value = 3.0f;
spatialFilter.Options[Option.FilterSmoothAlpha].Value = 0.5f;
spatialFilter.Options[Option.FilterSmoothDelta].Value = 18.0f;
temporalFilter.Options[Option.HolesFill].Value = 2;
temporalFilter.Options[Option.FilterSmoothAlpha].Value = 0.6f;
temporalFilter.Options[Option.FilterSmoothDelta].Value = 30.0f;
}
private void TestTimeInstantFromFile(string filepath, TemporalFilter.ValueReferenceType valRef, TemporalFilter.OperatorType expectedOperator, DateTime expDateTime)
{
// Getting the proxy
var xmlBytes = System.IO.File.ReadAllBytes(filepath);
var proxy = DeserialiseAndGetProxy(xmlBytes);
// Creating an object from the proxy
var testObject = new TemporalFilter(proxy);
// Asserting value reference
Assert.AreEqual(valRef, testObject.ValueReference);
// Asserting operator
Assert.AreEqual(expectedOperator, testObject.Operator);
// Expecting a time instant
var timeInstant = (Item_TimeInstant)testObject.Time;
AssertDateTime(expDateTime, timeInstant.Value);
}
private TemporalFilter SerialiseAndRead(TemporalFilter input)
{
// Get proxy
var filterProxy = input.ToXmlProxy("test_");
// Wrap the proxy in a GetObservationRequest
var requestProxy = new XsdNs.GetObservationType
{
temporalFilter = new XsdNs.GetObservationTypeTemporalFilter[] { filterProxy },
service = "SOS",
version = "2.0.0"
};
// Serialise the proxy
byte[] xmlBytes = XNeut.Helper.ToXmlBytes(requestProxy);
// XML validation
Validate(xmlBytes);
// Read XML data
var proxyIn = DeserialiseAndGetProxy(xmlBytes);
return(new TemporalFilter(proxyIn));
}
private void SetupProcessingBlock(Pipeline pipeline, Colorizer colorizer, DecimationFilter decimate, SpatialFilter spatial, TemporalFilter temp, HoleFillingFilter holeFill, ThresholdFilter threshold)
{
// Setup / start frame processing
processingBlock = new CustomProcessingBlock((f, src) =>
{
// We create a FrameReleaser object that would track
// all newly allocated .NET frames, and ensure deterministic finalization
// at the end of scope.
using (var releaser = new FramesReleaser())
{
using (var frames = pipeline.WaitForFrames().DisposeWith(releaser))
{
var processedFrames = frames
.ApplyFilter(decimate).DisposeWith(releaser)
.ApplyFilter(spatial).DisposeWith(releaser)
.ApplyFilter(temp).DisposeWith(releaser)
.ApplyFilter(holeFill).DisposeWith(releaser)
.ApplyFilter(colorizer).DisposeWith(releaser)
.ApplyFilter(threshold).DisposeWith(releaser);
// Send it to the next processing stage
src.FrameReady(processedFrames);
}
}
});
}
private CustomProcessingBlock SetupProcessingBlock(Pipeline pipeline, Colorizer colorizer, DecimationFilter decimate, SpatialFilter spatial, TemporalFilter temp, HoleFillingFilter holeFill, Align align_to)
{
CustomProcessingBlock processingBlock = null;
if (showType == imgType.color)
{
processingBlock = new CustomProcessingBlock((f, src) =>
{
using (var releaser = new FramesReleaser())
{
using (var frames = pipeline.WaitForFrames().DisposeWith(releaser))
{
var processedFrames = frames
.ApplyFilter(align_to).DisposeWith(releaser);
// Send it to the next processing stage
src.FramesReady(processedFrames);
}
}
});
}
else if (showType == imgType.mix)
{
// Setup / start frame processing
processingBlock = new CustomProcessingBlock((f, src) =>
{
using (var releaser = new FramesReleaser())
{
using (var frames = pipeline.WaitForFrames().DisposeWith(releaser))
{
var processedFrames = frames
.ApplyFilter(align_to).DisposeWith(releaser)
.ApplyFilter(decimate).DisposeWith(releaser)
.ApplyFilter(spatial).DisposeWith(releaser)
.ApplyFilter(temp).DisposeWith(releaser)
.ApplyFilter(holeFill).DisposeWith(releaser)
.ApplyFilter(colorizer).DisposeWith(releaser);
// Send it to the next processing stage
src.FramesReady(processedFrames);
}
}
});
}
return(processingBlock);
}
private void OnStartStreaming(PipelineProfile activeProfile)
{
pc = new PointCloud();
spatial = new SpatialFilter();
temporal = new TemporalFilter();
holeFilling = new HoleFillingFilter();
using (var profile = activeProfile.GetStream(stream))
{
if (profile == null)
{
Debug.LogWarningFormat("Stream {0} not in active profile", stream);
}
}
using (var profile = activeProfile.GetStream(Stream.Depth) as VideoStreamProfile)
{
intrinsics = profile.GetIntrinsics();
Assert.IsTrue(SystemInfo.SupportsTextureFormat(TextureFormat.RGFloat));
numParticles = (profile.Width - 1) * (profile.Height - 1) * 2;
vertices = new Vector3[profile.Width * profile.Height];
handle = GCHandle.Alloc(vertices, GCHandleType.Pinned);
verticesPtr = handle.AddrOfPinnedObject();
var indices = new int[(profile.Width - 1) * (profile.Height - 1) * 6];
var iIdx = 0;
for (int j = 0; j < profile.Height; j++)
{
for (int i = 0; i < profile.Width; i++)
{
if (i < profile.Width - 1 && j < profile.Height - 1)
{
var idx = i + j * profile.Width;
var y = profile.Width;
indices[iIdx++] = idx + 0;
indices[iIdx++] = idx + y;
indices[iIdx++] = idx + 1;
indices[iIdx++] = idx + 1;
indices[iIdx++] = idx + y;
indices[iIdx++] = idx + y + 1;
}
}
}
particleBuffer = new ComputeBuffer(numParticles, Marshal.SizeOf(typeof(VoxelParticle)));
vertexBuffer = new ComputeBuffer(vertices.Length, sizeof(float) * 3);
indicesBuffer = new ComputeBuffer(indices.Length, sizeof(int));
vertexBuffer.SetData(vertices);
indicesBuffer.SetData(indices);
renderer.SetBuffer("_VoxelBuffer", particleBuffer);
ResetParticle();
if (mesh != null)
{
Destroy(mesh);
}
mesh = new Mesh()
{
indexFormat = IndexFormat.UInt32,
};
mesh.MarkDynamic();
mesh.vertices = new Vector3[numParticles];
var newIdices = Enumerable.Range(0, numParticles).ToArray();
mesh.SetIndices(newIdices, MeshTopology.Points, 0, false);
mesh.bounds = new Bounds(Vector3.zero, Vector3.one * 10f);
GetComponent <MeshFilter>().sharedMesh = mesh;
}
RealSenseDevice.Instance.onNewSampleSet += OnFrames;
}
private void RunThread(CancellationToken token)
{
Intel.RealSense.PointCloud pc = new Intel.RealSense.PointCloud();
DecimationFilter dec_filter = new DecimationFilter();
SpatialFilter spat_filter = new SpatialFilter();
TemporalFilter temp_filter = new TemporalFilter();
dec_filter.Options[Option.FilterMagnitude].Value = DecimationMagnitude;
spat_filter.Options[Option.FilterMagnitude].Value = SpatialMagnitude;
spat_filter.Options[Option.FilterSmoothAlpha].Value = Math.Min(1.0f, (float)SpatialSmoothAlpha);
spat_filter.Options[Option.FilterSmoothDelta].Value = (float)SpatialSmoothDelta;
temp_filter.Options[Option.FilterSmoothAlpha].Value = Math.Min(1.0f, (float)TemporalSmoothAlpha);
temp_filter.Options[Option.FilterSmoothDelta].Value = (float)TemporalSmoothDelta;
List <ProcessingBlock> filters = new List <ProcessingBlock> {
dec_filter, spat_filter, temp_filter
};
Align align_to_depth = new Align(Stream.Depth);
var cfg = new Config();
cfg.EnableStream(Stream.Depth, 640, 480);
cfg.EnableStream(Stream.Color, 1280, 720, Format.Rgb8);
var pipeline = new Pipeline();
PipelineProfile pp = null;
try
{
pp = pipeline.Start(cfg);
}
catch (Exception e)
{
RhinoApp.WriteLine("RsToolkit: " + e.Message);
return;
}
while (!token.IsCancellationRequested)
{
try
{
using (var frames = pipeline.WaitForFrames())
{
var aligned = align_to_depth.Process <FrameSet>(frames).DisposeWith(frames);
var color = aligned.ColorFrame.DisposeWith(frames);
pc.MapTexture(color);
var filtered = aligned[Stream.Depth].DisposeWith(frames);
foreach (var filter in filters)
{
filtered = filter.Process(filtered).DisposeWith(frames);
}
Points points = pc.Process <Points>(filtered);
var vertices = new Point3f[points.Count];
var tex_coords = new Point2f[points.Count];
points.CopyVertices <Point3f>(vertices);
points.CopyTextureCoords <Point2f>(tex_coords);
Debug.Assert(vertices.Length == tex_coords.Length);
// ======== CULL INVALID POINTS ========
if (true)
{
var flags = new bool[vertices.Length];
int new_size = 0;
for (int i = 0; i < vertices.Length; ++i)
{
if (vertices[i].Z > 0.1)
{
flags[i] = true;
new_size++;
}
}
var new_vertices = new Point3f[new_size];
var new_tex_coords = new Point2f[new_size];
for (int i = 0, j = 0; i < vertices.Length; ++i)
{
if (flags[i])
{
new_vertices[j] = vertices[i];
new_tex_coords[j] = tex_coords[i];
++j;
}
}
vertices = new_vertices;
tex_coords = new_tex_coords;
}
// ======== TRANSFORM ========
if (m_xform.IsValid)
{
Parallel.For(0, vertices.Length - 1, (i) =>
{
vertices[i].Transform(m_xform);
});
}
// ======== CLIP TO BOX ========
if (m_clipping_box.IsValid &&
m_clipping_box.X.Length > 0 &&
m_clipping_box.Y.Length > 0 &&
m_clipping_box.Z.Length > 0)
{
Point3d box_centre = m_clipping_box.Plane.Origin;
double minx = m_clipping_box.X.Min + box_centre.X, maxx = m_clipping_box.X.Max + box_centre.X;
double miny = m_clipping_box.Y.Min + box_centre.Y, maxy = m_clipping_box.Y.Max + box_centre.Y;
double minz = m_clipping_box.Z.Min + box_centre.Z, maxz = m_clipping_box.Z.Max + box_centre.Z;
var flags = new bool[vertices.Length];
int new_size = 0;
for (int i = 0; i < vertices.Length; ++i)
{
if (
vertices[i].X <maxx && vertices[i].X> minx &&
vertices[i].Y <maxy && vertices[i].Y> miny &&
vertices[i].Z <maxz && vertices[i].Z> minz
)
{
flags[i] = true;
new_size++;
}
}
var new_vertices = new Point3f[new_size];
var new_tex_coords = new Point2f[new_size];
for (int i = 0, j = 0; i < vertices.Length; ++i)
{
if (flags[i])
{
new_vertices[j] = vertices[i];
new_tex_coords[j] = tex_coords[i];
++j;
}
}
vertices = new_vertices;
tex_coords = new_tex_coords;
}
Debug.Assert(vertices.Length == tex_coords.Length);
var point_colors = GetPointColors(color, tex_coords);
RPointCloud new_pointcloud = new RPointCloud();
new_pointcloud.AddRange(vertices.Select(x => new Point3d(x)), point_colors);
lock (m_pointcloud)
m_pointcloud = new_pointcloud;
}
}
catch (Exception e)
{
RhinoApp.WriteLine("RsToolkit: " + e.Message);
m_is_on = false;
break;
}
}
RhinoApp.WriteLine("RsToolkit: Task cancelled.");
if (pipeline != null)
{
pipeline.Stop();
}
}
/// <inheritdoc/>
public override ProcessingBlock Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
//Note: Maybe we should add checks for each read.
_ = reader.Read(); // Read start object
_ = reader.GetString(); // Read "Name"
_ = reader.Read(); // Read the ':'
var name = reader.GetString(); // Read the name
ProcessingBlock block;
switch (name)
{
case "Decimation Filter":
block = new DecimationFilter();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "FilterMagnitude"
_ = reader.Read(); // Read the ':'
block.Options[Option.FilterMagnitude].Value = reader.GetSingle();
_ = reader.Read(); // Read end object
break;
case "Spatial Filter":
block = new SpatialFilter();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "FilterMagnitude"
_ = reader.Read(); // Read the ':'
block.Options[Option.FilterMagnitude].Value = reader.GetSingle();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "FilterSmoothAlpha"
_ = reader.Read(); // Read the ':'
block.Options[Option.FilterSmoothAlpha].Value = reader.GetSingle();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "FilterSmoothDelta"
_ = reader.Read(); // Read the ':'
block.Options[Option.FilterSmoothDelta].Value = reader.GetSingle();
_ = reader.Read(); // Read end object
break;
case "Temporal Filter":
block = new TemporalFilter();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "FilterSmoothAlpha"
_ = reader.Read(); // Read the ':'
block.Options[Option.FilterSmoothAlpha].Value = reader.GetSingle();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "FilterSmoothDelta"
_ = reader.Read(); // Read the ':'
block.Options[Option.FilterSmoothDelta].Value = reader.GetSingle();
_ = reader.Read(); // Read end object
break;
case "Hole Filling Filter":
block = new HoleFillingFilter();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "HolesFill"
_ = reader.Read(); // Read the ':'
block.Options[Option.HolesFill].Value = reader.GetSingle();
_ = reader.Read(); // Read end object
break;
case "Threshold Filter":
block = new ThresholdFilter();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "MinDistance"
_ = reader.Read(); // Read the ':'
block.Options[Option.MinDistance].Value = reader.GetSingle();
_ = reader.Read(); // Read the ','
_ = reader.GetString(); // Read "MaxDistance"
_ = reader.Read(); // Read the ':'
block.Options[Option.MaxDistance].Value = reader.GetSingle();
_ = reader.Read(); // Read end object
break;
default:
throw new NotSupportedException($"The filter {name} is not supported in this converter");
}
return(block);
}
public void GetObsReq_122_Create()
{
// Creating request object
var testObject = new GetObservationRequest();
testObject.FeaturesOfInterest.Add("myfeature");
testObject.ObservedProperties.Add("myproperty");
// Adding a data record
var extensionObj = new Item_DataRecord
{
{ "MyMeasurement", new Item_Measurement("s", 0.453) }
};
testObject.Items.Add(extensionObj);
// Adding temporal filters
var baseTime = DateTime.Now.ToUniversalTime();
var tempFilter1 = new TemporalFilter(
TemporalFilter.ValueReferenceType.ResultTime,
TemporalFilter.OperatorType.During,
new Item_TimeRange(baseTime, baseTime.AddHours(2))
);
testObject.TemporalFilters.Add(tempFilter1);
var tempFilter2 = new TemporalFilter(
TemporalFilter.ValueReferenceType.PhenomenonTime,
TemporalFilter.OperatorType.Before,
new Item_TimeInstant(baseTime)
);
testObject.TemporalFilters.Add(tempFilter2);
// Serialising, validating and deserialising request object
var xmlBytes = testObject.ToXmlBytes();
Validate(xmlBytes);
var testObjectIn = new GetObservationRequest(xmlBytes);
// Asserting
Assert.AreEqual(1, testObjectIn.FeaturesOfInterest.Count);
Assert.AreEqual(1, testObjectIn.ObservedProperties.Count);
Assert.IsTrue(testObjectIn.FeaturesOfInterest.Contains("myfeature"));
Assert.IsTrue(testObjectIn.ObservedProperties.Contains("myproperty"));
// Asserting extension (a data record)
Assert.AreEqual(1, testObjectIn.Items.Count);
var extension = testObjectIn.Items[0];
var measurementItem = (Item_Measurement)extension["MyMeasurement"];
Assert.AreEqual(0.453, measurementItem.Value, 0.0001);
Assert.AreEqual(2, testObject.TemporalFilters.Count);
// Asserting temporal filter 1 (the assertion has a low coverage because filters are tested elsewhere)
var filter1 = testObjectIn.TemporalFilters[0];
Assert.AreEqual(TemporalFilter.ValueReferenceType.ResultTime, filter1.ValueReference);
Assert.AreEqual(TemporalFilter.OperatorType.During, filter1.Operator);
// Asserting temporal filter 2
var filter2 = testObjectIn.TemporalFilters[1];
Assert.AreEqual(TemporalFilter.ValueReferenceType.PhenomenonTime, filter2.ValueReference);
Assert.AreEqual(TemporalFilter.OperatorType.Before, filter2.Operator);
}
