private void ExchangeModel(LandmarkModelType modelType)
{
selectedModelType = modelType;
var nnModel = selectedModelType == LandmarkModelType.Lite ? resource.liteModel : resource.fullModel;
var model = ModelLoader.Load(nnModel);
worker = model.CreateWorker();
}
public PoseLandMarker(PoseLandMarkResource resource, LandmarkModelType modelType)
{
this.resource = resource;
preBuffer = new ComputeBuffer(IMAGE_SIZE * IMAGE_SIZE * 3, sizeof(float));
outputLandmarkBuffer = new ComputeBuffer(KEYPOINT_COUNT + 1, sizeof(float) * 4);
outputLandmarkWorldBuffer = new ComputeBuffer(KEYPOINT_COUNT + 1, sizeof(float) * 4);
segmentRT = new RenderTexture(SEGMENT_IMAGE_SIZE, SEGMENT_IMAGE_SIZE, 0, RenderTextureFormat.ARGB32);
ExchangeModel(modelType);
}
private void RunModel(Texture source, LandmarkModelType modelType, float poseThreshold = 0.75f, float iouThreshold = 0.3f)
{
// letterboxing scale factor
var scale = new Vector2(Mathf.Max((float)source.height / source.width, 1),
Mathf.Max((float)source.width / source.height, 1));
var cs = resource.process;
// Letter box Image
cs.SetInt("_LetterBoxWidth", DETECTION_INPUT_IMAGE_SIZE);
cs.SetVector("_LetterBoxScale", scale);
cs.SetTexture(0, "_LetterBoxInput", source);
cs.SetTexture(0, "_LetterBoxOutput", letterboxTexture);
cs.DispatchThreads(0, DETECTION_INPUT_IMAGE_SIZE, DETECTION_INPUT_IMAGE_SIZE, 1);
// Predict pose
detector.ProcessImage(letterboxTexture, poseThreshold, iouThreshold);
// Update pose region from detected image
cs.SetFloat("_DeltaTime", Time.deltaTime);
cs.SetInt("_UpperBodyOnly", 0);
cs.SetBuffer(1, "_Poses", detector.OutputBuffer);
cs.SetBuffer(1, "_PoseCount", detector.CountBuffer);
cs.SetBuffer(1, "_PoseRegions", poseRegionBuffer);
cs.DispatchThreads(1, 1, 1, 1);
// Scale and pad to letter-box image and crop pose region from source texture
cs.SetTexture(2, "_SourceTexture", source);
cs.SetBuffer(2, "_CropRegions", poseRegionBuffer);
cs.SetTexture(2, "_CroppedTexture", croppedTexture);
cs.DispatchThreads(2, LANDMARK_INPUT_IMAGE_SIZE, LANDMARK_INPUT_IMAGE_SIZE, 1);
landmarker.ProcessImage(croppedTexture, modelType);
// map cordinates from croped letterbox image to source image
cs.SetInt("_KeyPointCount", landmarker.KeypointCount);
cs.SetFloat("_PostDeltaTime", Time.deltaTime);
cs.SetBuffer(3, "_PostInput", landmarker.OutputLandmarkBuffer);
cs.SetBuffer(3, "_PostInputWorld", landmarker.OutputLandmarkWorldBuffer);
cs.SetBuffer(3, "_PostRegions", poseRegionBuffer);
cs.SetBuffer(3, "_PostDeltaOutput", deltaOutputBuffer);
cs.SetBuffer(3, "_PostDeltaOutputWorld", deltaOutputWorldBuffer);
cs.SetBuffer(3, "_PostOutput", outputBuffer);
cs.SetBuffer(3, "_PostOutputWorld", outputWorldBuffer);
cs.DispatchThreads(3, 1, 1, 1);
}
public BlazePoseDetector(BlazePoseResource resource, LandmarkModelType modelType)
{
this.resource = resource;
detector = new PoseDetector(resource.detectionResource);
landmarker = new PoseLandMarker(resource.landmarkResource, modelType);
letterboxTexture = RTUtil.NewUAV(DETECTION_INPUT_IMAGE_SIZE, DETECTION_INPUT_IMAGE_SIZE, 0, RenderTextureFormat.ARGB32, GraphicsFormat.R8G8B8A8_UNorm);
croppedTexture = RTUtil.NewUAV(LANDMARK_INPUT_IMAGE_SIZE, LANDMARK_INPUT_IMAGE_SIZE, 0, RenderTextureFormat.ARGB32, GraphicsFormat.R8G8B8A8_UNorm);
poseRegionBuffer = new ComputeBuffer(1, sizeof(float) * 24);
deltaOutputBuffer = new ComputeBuffer(landmarker.KeypointCount, sizeof(float) * 4);
deltaOutputWorldBuffer = new ComputeBuffer(landmarker.KeypointCount, sizeof(float) * 4);
// feature 33 + human exist flag 1
outputBuffer = new ComputeBuffer(landmarker.KeypointCount + 1, sizeof(float) * 4);
outputWorldBuffer = new ComputeBuffer(landmarker.KeypointCount + 1, sizeof(float) * 4);
}
private void RunModel(Texture source, LandmarkModelType modelType)
{
if (modelType != selectedModelType)
{
ExchangeModel(modelType);
}
var pre = resource.preprocess;
pre.SetInt("_Size", IMAGE_SIZE);
pre.SetTexture(0, "_Image", source);
pre.SetBuffer(0, "_Tensor", preBuffer);
pre.DispatchThreads(0, IMAGE_SIZE, IMAGE_SIZE, 1);
using (var tensor = new Tensor(1, IMAGE_SIZE, IMAGE_SIZE, 3, preBuffer))
worker.Execute(tensor);
var poseFlagBuffer = worker.CopyOutputToBuffer("Identity_1", 1);
var landmarkBuffer = worker.CopyOutputToBuffer("Identity", KEYPOINT_BUFFER_LEN);
var landmarkWorldBuffer = worker.CopyOutputToBuffer("Identity_4", WORLD_LD_LEN);
var post = resource.postprocess;
post.SetInt("_KeypointCount", KEYPOINT_COUNT);
post.SetBuffer(0, "_PoseFlag", poseFlagBuffer);
post.SetBuffer(0, "_Landmark", landmarkBuffer);
post.SetBuffer(0, "_LandmarkWorld", landmarkWorldBuffer);
post.SetBuffer(0, "_Output", outputLandmarkBuffer);
post.SetBuffer(0, "_OutputWorld", outputLandmarkWorldBuffer);
post.DispatchThreads(0, 1, 1, 1);
var segTemp = worker.CopyOutputToTempRT("Identity_2", SEGMENT_IMAGE_SIZE, SEGMENT_IMAGE_SIZE, RenderTextureFormat.ARGB32);
Graphics.Blit(segTemp, segmentRT);
RenderTexture.ReleaseTemporary(segTemp);
}
public void ProcessImage(Texture source, LandmarkModelType modelType) => RunModel(source, modelType);
