/// <inheritdoc/>
protected override void Setup()
{
var myCamera = perceptionCamera.GetComponent <Camera>();
var width = myCamera.pixelWidth;
var height = myCamera.pixelHeight;
if (labelConfig == null)
{
throw new InvalidOperationException(
"SemanticSegmentationLabeler's LabelConfig must be assigned");
}
m_AsyncAnnotations = new Dictionary <int, AsyncAnnotation>();
var renderTextureDescriptor = new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8);
if (targetTexture != null)
{
targetTexture.descriptor = renderTextureDescriptor;
}
else
{
m_TargetTextureOverride = new RenderTexture(renderTextureDescriptor);
}
targetTexture.Create();
targetTexture.name = "Labeling";
#if HDRP_PRESENT
var gameObject = perceptionCamera.gameObject;
var customPassVolume = gameObject.GetComponent <CustomPassVolume>() ?? gameObject.AddComponent <CustomPassVolume>();
customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
customPassVolume.isGlobal = true;
m_SemanticSegmentationPass = new SemanticSegmentationPass(myCamera, targetTexture, labelConfig)
{
name = "Labeling Pass"
};
customPassVolume.customPasses.Add(m_SemanticSegmentationPass);
#endif
#if URP_PRESENT
perceptionCamera.AddScriptableRenderPass(new SemanticSegmentationUrpPass(myCamera, targetTexture, labelConfig));
#endif
var specs = labelConfig.labelEntries.Select((l) => new SemanticSegmentationSpec()
{
label_name = l.label,
pixel_value = l.color
}).ToArray();
m_SemanticSegmentationAnnotationDefinition = DatasetCapture.RegisterAnnotationDefinition(
"semantic segmentation",
specs,
"pixel-wise semantic segmentation label",
"PNG",
id: Guid.Parse(annotationId));
m_SemanticSegmentationTextureReader = new RenderTextureReader <Color32>(targetTexture, myCamera,
(frameCount, data, tex) => OnSemanticSegmentationImageRead(frameCount, data));
}
void SetupInstanceSegmentation()
{
var myCamera = GetComponent <Camera>();
var width = myCamera.pixelWidth;
var height = myCamera.pixelHeight;
m_InstanceSegmentationTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
m_InstanceSegmentationTexture.filterMode = FilterMode.Point;
m_InstanceSegmentationTexture.name = "InstanceSegmentation";
m_RenderedObjectInfoGenerator = new RenderedObjectInfoGenerator();
#if HDRP_PRESENT || URP_PRESENT
#if HDRP_PRESENT
var customPassVolume = this.GetComponent <CustomPassVolume>() ?? gameObject.AddComponent <CustomPassVolume>();
customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
customPassVolume.isGlobal = true;
m_InstanceSegmentationPass = new InstanceSegmentationPass()
{
name = "Instance segmentation pass",
targetCamera = GetComponent <Camera>(),
targetTexture = m_InstanceSegmentationTexture
};
m_InstanceSegmentationPass.EnsureInit();
customPassVolume.customPasses.Add(m_InstanceSegmentationPass);
m_LensDistortionPass = new LensDistortionPass(GetComponent <Camera>(), m_InstanceSegmentationTexture)
{
name = "Instance Segmentation Lens Distortion Pass"
};
m_LensDistortionPass.EnsureInit();
customPassVolume.customPasses.Add(m_LensDistortionPass);
m_fLensDistortionEnabled = true;
#elif URP_PRESENT
m_InstanceSegmentationPass = new InstanceSegmentationUrpPass(myCamera, m_InstanceSegmentationTexture);
AddScriptableRenderPass(m_InstanceSegmentationPass);
// Lens Distortion
m_LensDistortionPass = new LensDistortionUrpPass(myCamera, m_InstanceSegmentationTexture);
AddScriptableRenderPass(m_LensDistortionPass);
m_fLensDistortionEnabled = true;
#endif
m_LensDistortionPass.m_LensDistortionCrossPipelinePass.lensDistortionOverride =
m_LensDistortionIntensityOverride;
#endif
m_InstanceSegmentationReader = new RenderTextureReader <Color32>(m_InstanceSegmentationTexture, myCamera, (frameCount, data, tex) =>
{
InstanceSegmentationImageReadback?.Invoke(frameCount, data, tex);
if (RenderedObjectInfosCalculated != null)
{
m_RenderedObjectInfoGenerator.Compute(data, tex.width, BoundingBoxOrigin.TopLeft, out var renderedObjectInfos, Allocator.Temp);
RenderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
renderedObjectInfos.Dispose();
}
/// <inheritdoc/>
protected override void Cleanup()
{
m_SemanticSegmentationTextureReader?.WaitForAllImages();
m_SemanticSegmentationTextureReader?.Dispose();
m_SemanticSegmentationTextureReader = null;
if (m_TargetTextureOverride != null)
{
m_TargetTextureOverride.Release();
}
m_TargetTextureOverride = null;
}
void SetupInstanceSegmentation()
{
var myCamera = GetComponent <Camera>();
var width = myCamera.pixelWidth;
var height = myCamera.pixelHeight;
m_InstanceSegmentationTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
m_InstanceSegmentationTexture.filterMode = FilterMode.Point;
m_InstanceSegmentationTexture.name = "InstanceSegmentation";
m_RenderedObjectInfoGenerator = new RenderedObjectInfoGenerator();
#if HDRP_PRESENT || URP_PRESENT
#if HDRP_PRESENT
var customPassVolume = this.GetComponent <CustomPassVolume>() ?? gameObject.AddComponent <CustomPassVolume>();
customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
customPassVolume.isGlobal = true;
m_InstanceSegmentationPass = new InstanceSegmentationPass()
{
name = "Instance segmentation pass",
targetCamera = GetComponent <Camera>(),
targetTexture = m_InstanceSegmentationTexture
};
m_InstanceSegmentationPass.EnsureInit();
customPassVolume.customPasses.Add(m_InstanceSegmentationPass);
m_LensDistortionPass = new LensDistortionPass(GetComponent <Camera>(), m_InstanceSegmentationTexture)
{
name = "Instance Segmentation Lens Distortion Pass"
};
m_LensDistortionPass.EnsureInit();
customPassVolume.customPasses.Add(m_LensDistortionPass);
#elif URP_PRESENT
m_InstanceSegmentationPass = new InstanceSegmentationUrpPass(myCamera, m_InstanceSegmentationTexture);
AddScriptableRenderPass(m_InstanceSegmentationPass);
// Lens Distortion
m_LensDistortionPass = new LensDistortionUrpPass(myCamera, m_InstanceSegmentationTexture);
AddScriptableRenderPass(m_LensDistortionPass);
#endif
m_LensDistortionPass.lensDistortionCrossPipelinePass.lensDistortionOverride =
m_LensDistortionIntensityOverride;
#endif
m_InstanceSegmentationReader = new RenderTextureReader <Color32>(m_InstanceSegmentationTexture);
}
// Start is called before the first frame update
void Awake()
{
//CaptureOptions.useAsyncReadbackIfSupported = false;
m_EgoMarker = this.GetComponentInParent <Ego>();
var ego = m_EgoMarker == null?SimulationManager.RegisterEgo("") : m_EgoMarker.EgoHandle;
SensorHandle = SimulationManager.RegisterSensor(ego, "camera", description, period, startTime);
var myCamera = GetComponent <Camera>();
var width = myCamera.pixelWidth;
var height = myCamera.pixelHeight;
if ((produceSegmentationImages || produceObjectCountAnnotations || produceBoundingBoxAnnotations) && LabelingConfiguration == null)
{
Debug.LogError("LabelingConfiguration must be set if producing ground truth data");
produceSegmentationImages = false;
produceObjectCountAnnotations = false;
produceBoundingBoxAnnotations = false;
}
segmentationTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
segmentationTexture.name = "Segmentation";
labelingTexture = new RenderTexture(new RenderTextureDescriptor(width, height, GraphicsFormat.R8G8B8A8_UNorm, 8));
labelingTexture.name = "Labeling";
#if HDRP_PRESENT
var customPassVolume = this.GetComponent <CustomPassVolume>() ?? gameObject.AddComponent <CustomPassVolume>();
customPassVolume.injectionPoint = CustomPassInjectionPoint.BeforeRendering;
customPassVolume.isGlobal = true;
m_SegmentationPass = new InstanceSegmentationPass()
{
name = "Segmentation Pass",
targetCamera = myCamera,
targetTexture = segmentationTexture
};
m_SegmentationPass.EnsureInit();
m_SemanticSegmentationPass = new SemanticSegmentationPass(myCamera, labelingTexture, LabelingConfiguration)
{
name = "Labeling Pass"
};
SetupPasses(customPassVolume);
#endif
#if URP_PRESENT
instanceSegmentationUrpPass = new InstanceSegmentationUrpPass(myCamera, segmentationTexture);
semanticSegmentationUrpPass = new SemanticSegmentationUrpPass(myCamera, labelingTexture, LabelingConfiguration);
#endif
if (produceSegmentationImages)
{
var specs = LabelingConfiguration.LabelEntries.Select((l) => new SemanticSegmentationSpec()
{
label_id = l.id,
label_name = l.label,
pixel_value = l.value
}).ToArray();
m_SegmentationAnnotationDefinition = SimulationManager.RegisterAnnotationDefinition("semantic segmentation", specs, "pixel-wise semantic segmentation label", "PNG");
m_ClassLabelingTextureReader = new RenderTextureReader <short>(labelingTexture, myCamera,
(frameCount, data, tex) => OnSemanticSegmentationImageRead(frameCount, data));
}
if (produceObjectCountAnnotations || produceBoundingBoxAnnotations || produceRenderedObjectInfoMetric)
{
var labelingMetricSpec = LabelingConfiguration.LabelEntries.Select((l) => new ObjectCountSpec()
{
label_id = l.id,
label_name = l.label,
}).ToArray();
if (produceObjectCountAnnotations)
{
m_ObjectCountMetricDefinition = SimulationManager.RegisterMetricDefinition("object count", labelingMetricSpec, "Counts of objects for each label in the sensor's view", id: new Guid(objectCountId));
}
if (produceBoundingBoxAnnotations)
{
m_BoundingBoxAnnotationDefinition = SimulationManager.RegisterAnnotationDefinition("bounding box", labelingMetricSpec, "Bounding box for each labeled object visible to the sensor", id: new Guid(boundingBoxId));
}
if (produceRenderedObjectInfoMetric)
{
m_RenderedObjectInfoMetricDefinition = SimulationManager.RegisterMetricDefinition("rendered object info", labelingMetricSpec, "Information about each labeled object visible to the sensor", id: new Guid(renderedObjectInfoId));
}
m_RenderedObjectInfoGenerator = new RenderedObjectInfoGenerator(LabelingConfiguration);
World.DefaultGameObjectInjectionWorld.GetExistingSystem <GroundTruthLabelSetupSystem>().Activate(m_RenderedObjectInfoGenerator);
m_SegmentationReader = new RenderTextureReader <uint>(segmentationTexture, myCamera, (frameCount, data, tex) =>
{
if (segmentationImageReceived != null)
{
segmentationImageReceived(frameCount, data);
}
m_RenderedObjectInfoGenerator.Compute(data, tex.width, boundingBoxOrigin, out var renderedObjectInfos, out var classCounts, Allocator.Temp);
using (s_RenderedObjectInfosCalculatedEvent.Auto())
renderedObjectInfosCalculated?.Invoke(frameCount, renderedObjectInfos);
if (produceObjectCountAnnotations)
{
OnObjectCountsReceived(classCounts, LabelingConfiguration.LabelEntries, frameCount);
}
if (produceBoundingBoxAnnotations)
{
ProduceBoundingBoxesAnnotation(renderedObjectInfos, LabelingConfiguration.LabelEntries, frameCount);
}
if (produceRenderedObjectInfoMetric)
{
ProduceRenderedObjectInfoMetric(renderedObjectInfos, frameCount);
}
});