async void Start()
{
webCamScreen = GameObject.Find("WebCamScreen");
if (useGPU)
{
gpuResources = GpuResources.Create().ConsumeValue();
gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(gpuResources);
}
#if UNITY_EDITOR
var resourceManager = LocalAssetManager.Instance;
#else
var resourceManager = AssetBundleManager.Instance;
#endif
ResourceUtil.InitializeResourceManager(resourceManager);
try {
await resourceManager.LoadAllAssetsAsync();
IsAssetLoaded = true;
} catch (Exception e) {
Debug.LogError(e);
IsAssetLoadFailed = true;
}
}
private GlContext GetGlContext()
{
using (var glCalculatorHelper = new GlCalculatorHelper()) {
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
return(glCalculatorHelper.GetGlContext());
}
}
public void InitializeForTest_ShouldInitialize()
{
var glCalculatorHelper = new GlCalculatorHelper();
Assert.False(glCalculatorHelper.Initialized());
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
Assert.True(glCalculatorHelper.Initialized());
}
public void InitializeForTest_ShouldInitialize()
{
using (var glCalculatorHelper = new GlCalculatorHelper()) {
Assert.False(glCalculatorHelper.Initialized());
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
Assert.True(glCalculatorHelper.Initialized());
}
}
public void framebuffer_ShouldReturnGLName()
{
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
// default frame buffer
Assert.AreEqual(glCalculatorHelper.framebuffer, 0);
}
public void RunInGlContext_ShouldReturnInternal_When_FunctionReturnsInternal()
{
using (var glCalculatorHelper = new GlCalculatorHelper()) {
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
var status = glCalculatorHelper.RunInGlContext(() => { return(Status.Build(Status.StatusCode.Internal, "error")); });
Assert.AreEqual(status.code, Status.StatusCode.Internal);
}
}
public void RunInGlContext_ShouldReturnOk_When_FunctionReturnsOk()
{
using (var glCalculatorHelper = new GlCalculatorHelper()) {
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
var status = glCalculatorHelper.RunInGlContext(() => { return(Status.Ok()); });
Assert.True(status.ok);
}
}
public void RunInGlContext_ShouldReturnInternal_When_FunctionThrows()
{
using (var glCalculatorHelper = new GlCalculatorHelper())
{
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
var status = glCalculatorHelper.RunInGlContext((GlCalculatorHelper.GlFunction)(() => { throw new Exception("Function Throws"); }));
Assert.AreEqual(Status.StatusCode.Internal, status.Code());
}
}
public void Framebuffer_ShouldReturnGLName()
{
using (var glCalculatorHelper = new GlCalculatorHelper())
{
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
// default frame buffer
Assert.AreEqual(0, glCalculatorHelper.framebuffer);
}
}
public void RunInGlContext_ShouldReturnFailedPreCondition_When_FunctionThrows()
{
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
var status = glCalculatorHelper.RunInGlContext(() => { throw new InvalidProgramException(); });
Assert.AreEqual(status.code, Status.StatusCode.FailedPrecondition);
}
public static IEnumerator Initialize()
{
lock (setupLock) {
if (isInitialized)
{
Logger.LogWarning(TAG, "Already set up");
yield break;
}
#if UNITY_ANDROID && !UNITY_EDITOR
isContextInitialized = false;
PluginCallback callback = GetCurrentContext;
var fp = Marshal.GetFunctionPointerForDelegate(callback);
GL.IssuePluginEvent(fp, 1);
#else
isContextInitialized = true;
#endif
var count = 1000;
yield return(new WaitUntil(() => {
return --count < 0 || isContextInitialized;
}));
if (!isContextInitialized)
{
throw new TimeoutException("Failed to get GlContext");
}
#if UNITY_ANDROID
if (currentContext == IntPtr.Zero)
{
Logger.LogWarning(TAG, "EGL context is not found, so MediaPipe won't share their EGL contexts with Unity");
}
else
{
Logger.LogVerbose(TAG, $"EGL context is found: {currentContext}");
}
#endif
try {
Logger.LogInfo(TAG, "Initializing GpuResources...");
gpuResources = GpuResources.Create(currentContext).Value();
Logger.LogInfo(TAG, "Initializing GlCalculatorHelper...");
glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(gpuResources);
isInitialized = true;
} catch (Exception e) {
Logger.LogException(e);
Logger.LogError(TAG, "Failed to create GpuResources. If your native library is built for CPU, change 'Preferable Inference Mode' to CPU from the Inspector Window for Bootstrap");
}
}
}
public void GetCurrent_ShouldReturnCurrentContext_When_CalledInGlContext()
{
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
glCalculatorHelper.RunInGlContext(() => {
var glContext = GlContext.GetCurrent();
Assert.NotNull(glContext);
Assert.True(glContext.IsCurrent());
return(Status.Ok());
}).AssertOk();
}
public void CreateSourceTexture_ShouldFail_When_ImageFrameFormatIsInvalid()
{
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
var imageFrame = new ImageFrame(ImageFormat.Format.SBGRA, 32, 24);
var status = glCalculatorHelper.RunInGlContext(() => {
var texture = glCalculatorHelper.CreateSourceTexture(imageFrame);
texture.Dispose();
return(Status.Ok());
});
Assert.AreEqual(status.code, Status.StatusCode.FailedPrecondition);
}
public void GetGlContext_ShouldReturnCurrentContext()
{
using (var glCalculatorHelper = new GlCalculatorHelper()) {
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
var glContext = glCalculatorHelper.GetGlContext();
#if UNITY_EDITOR_LINUX || UNITY_STANDALONE_LINUX || UNITY_ANDROID
Assert.AreNotEqual(glContext.eglContext, IntPtr.Zero);
#elif UNITY_STANDALONE_OSX
Assert.AreNotEqual(glContext.nsglContext, IntPtr.Zero);
#elif UNITY_IOS
Assert.AreNotEqual(glContext.eaglContext, IntPtr.Zero);
#endif
}
}
private GlContext GetGlContext()
{
GlContext glContext = null;
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
glCalculatorHelper.RunInGlContext(() => {
glContext = GlContext.GetCurrent();
return(Status.Ok());
}).AssertOk();
return(glContext);
}
public void GetCurrent_ShouldReturnCurrentContext_When_CalledInGlContext()
{
using (var glCalculatorHelper = new GlCalculatorHelper())
{
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
glCalculatorHelper.RunInGlContext(() =>
{
using (var glContext = GlContext.GetCurrent())
{
Assert.NotNull(glContext);
Assert.True(glContext.IsCurrent());
}
}).AssertOk();
}
}
public void CreateDestinationTexture_ShouldReturnGlTexture_When_GpuBufferFormatIsValid()
{
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
var status = glCalculatorHelper.RunInGlContext(() => {
var glTexture = glCalculatorHelper.CreateDestinationTexture(32, 24, GpuBufferFormat.kBGRA32);
Assert.AreEqual(glTexture.width, 32);
Assert.AreEqual(glTexture.height, 24);
return(Status.Ok());
});
Assert.True(status.ok);
}
void ExpectSetSurfaceOk(IntPtr surface)
{
var eglSurfaceHolder = new EglSurfaceHolder();
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
var status = glCalculatorHelper.RunInGlContext(() => {
var glContext = GlContext.GetCurrent();
eglSurfaceHolder.SetSurface(surface, glContext);
return(Status.Ok());
});
Assert.True(status.ok);
}
public void CreateDestinationTexture_ShouldReturnGlTexture_When_GpuBufferFormatIsValid()
{
using (var glCalculatorHelper = new GlCalculatorHelper())
{
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
var status = glCalculatorHelper.RunInGlContext(() =>
{
var glTexture = glCalculatorHelper.CreateDestinationTexture(32, 24, GpuBufferFormat.kBGRA32);
Assert.AreEqual(32, glTexture.width);
Assert.AreEqual(24, glTexture.height);
});
Assert.True(status.Ok());
}
}
void InitializeGpuHelper(int eventId)
{
// context is EGL_NO_CONTEXT if the graphics API is not OpenGL ES
var context = Egl.getCurrentContext();
if (context == IntPtr.Zero)
{
Debug.LogWarning("No EGL Context Found");
}
else
{
Debug.Log($"EGL Context Found ({context})");
}
gpuResources = GpuResources.Create(context).ConsumeValueOrDie();
gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(gpuResources);
}
/// <summary>
/// This method must be called (only) once before calling StartRun.
/// `graph` and `gpuHelper` (if useGPU is true) are initialized here.
/// If the config is invalid, it throws an error.
/// </summary>
public void Initialize()
{
if (config == null)
{
throw new InvalidOperationException("config is missing");
}
graph = new CalculatorGraph(config.text);
if (shouldUseGPU())
{
var gpuResources = new StatusOrGpuResources().ConsumeValue();
graph.SetGpuResources(gpuResources).AssertOk();
gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(graph.GetGpuResources());
}
}
public void CreateSourceTexture_ShouldReturnGlTexture_When_CalledWithImageFrame()
{
var glCalculatorHelper = new GlCalculatorHelper();
glCalculatorHelper.InitializeForTest(GpuResources.Create().ConsumeValueOrDie());
var imageFrame = new ImageFrame(ImageFormat.Format.SRGBA, 32, 24);
var status = glCalculatorHelper.RunInGlContext(() => {
var texture = glCalculatorHelper.CreateSourceTexture(imageFrame);
Assert.AreEqual(texture.width, 32);
Assert.AreEqual(texture.height, 24);
texture.Dispose();
return(Status.Ok());
});
Assert.True(status.ok);
}
public void CreateSourceTexture_ShouldFail_When_ImageFrameFormatIsInvalid()
{
using (var glCalculatorHelper = new GlCalculatorHelper())
{
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
using (var imageFrame = new ImageFrame(ImageFormat.Types.Format.Sbgra, 32, 24))
{
var status = glCalculatorHelper.RunInGlContext(() =>
{
using (var texture = glCalculatorHelper.CreateSourceTexture(imageFrame))
{
texture.Release();
}
});
Assert.AreEqual(Status.StatusCode.FailedPrecondition, status.Code());
status.Dispose();
}
}
}
void SetupGpuResources()
{
if (gpuResources != null)
{
Debug.Log("Gpu resources are already initialized");
return;
}
// TODO: have to wait for currentContext to be initialized.
if (currentContext == IntPtr.Zero)
{
Debug.LogWarning("No EGL Context Found");
}
else
{
Debug.Log($"EGL Context Found ({currentContext})");
}
gpuResources = GpuResources.Create(currentContext).ConsumeValueOrDie();
gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(gpuResources);
}
public void CreateSourceTexture_ShouldReturnGlTexture_When_CalledWithImageFrame()
{
using (var glCalculatorHelper = new GlCalculatorHelper())
{
glCalculatorHelper.InitializeForTest(GpuResources.Create().Value());
using (var imageFrame = new ImageFrame(ImageFormat.Types.Format.Srgba, 32, 24))
{
var status = glCalculatorHelper.RunInGlContext(() =>
{
var texture = glCalculatorHelper.CreateSourceTexture(imageFrame);
Assert.AreEqual(32, texture.width);
Assert.AreEqual(24, texture.height);
texture.Dispose();
});
Assert.True(status.Ok());
status.Dispose();
}
}
}
/// <summary>
/// Initialize GPU resources.
/// If it finishes successfully, <see cref="IsInitialized" /> will be set to <c>true</c>.
/// </summary>
/// <remarks>
/// If <see cref="IsInitialized" /> is <c>true</c>, it will do nothing.
/// Before the application exits, don't forget to call <see cref="Shutdown" />.
/// </remarks>
public static IEnumerator Initialize()
{
lock (_SetupLock)
{
if (IsInitialized)
{
Logger.LogInfo(_TAG, "Already initialized");
yield break;
}
#if UNITY_ANDROID
_IsContextInitialized = SystemInfo.graphicsDeviceType != GraphicsDeviceType.OpenGLES3;
if (!_IsContextInitialized)
{
PluginCallback callback = GetCurrentContext;
var fp = Marshal.GetFunctionPointerForDelegate(callback);
GL.IssuePluginEvent(fp, 1);
}
#else
_IsContextInitialized = true;
#endif
var count = 100;
yield return(new WaitUntil(() =>
{
return --count < 0 || _IsContextInitialized;
}));
if (!_IsContextInitialized)
{
Logger.LogError(_TAG, "Failed to get GlContext");
yield break;
}
#if UNITY_ANDROID
if (_CurrentContext == IntPtr.Zero)
{
Logger.LogWarning(_TAG, "EGL context is not found, so MediaPipe won't share their EGL contexts with Unity");
}
else
{
Logger.LogVerbose(_TAG, $"EGL context is found: {_CurrentContext}");
}
#endif
try
{
Logger.LogInfo(_TAG, "Initializing GpuResources...");
var statusOrGpuResources = GpuResources.Create(_CurrentContext);
statusOrGpuResources.status.AssertOk();
GpuResources = statusOrGpuResources.Value();
Logger.LogInfo(_TAG, "Initializing GlCalculatorHelper...");
GlCalculatorHelper = new GlCalculatorHelper();
GlCalculatorHelper.InitializeForTest(GpuResources);
IsInitialized = true;
}
catch (EntryPointNotFoundException e)
{
Logger.LogException(e);
Logger.LogError(_TAG, "Failed to create GpuResources. Did you build libraries with GPU enabled?");
}
catch (Exception e)
{
Logger.LogException(e);
}
}
}
IEnumerator RunGraph()
{
yield return(WaitForAssets());
if (IsAssetLoadFailed)
{
Debug.LogError("Failed to load assets. Stopping...");
yield break;
}
yield return(WaitForGraph());
if (graphPrefab == null)
{
Debug.LogWarning("No graph is set. Stopping...");
yield break;
}
var webCamScreenController = webCamScreen.GetComponent <WebCamScreenController>();
yield return(WaitForCamera(webCamScreenController));
if (!webCamScreenController.isPlaying)
{
Debug.LogWarning("WebCamDevice is not working. Stopping...");
yield break;
}
graphContainer = Instantiate(graphPrefab);
var graph = graphContainer.GetComponent <IDemoGraph <TextureFrame> >();
if (useGPU)
{
// TODO: have to wait for currentContext to be initialized.
if (currentContext == IntPtr.Zero)
{
Debug.LogWarning("No EGL Context Found");
}
else
{
Debug.Log($"EGL Context Found ({currentContext})");
}
gpuResources = GpuResources.Create(currentContext).ConsumeValueOrDie();
gpuHelper = new GlCalculatorHelper();
gpuHelper.InitializeForTest(gpuResources);
graph.Initialize(gpuResources, gpuHelper);
}
else
{
graph.Initialize();
}
graph.StartRun(webCamScreenController.GetScreen()).AssertOk();
while (true)
{
yield return(new WaitForEndOfFrame());
var nextFrameRequest = webCamScreenController.RequestNextFrame();
yield return(nextFrameRequest);
var nextFrame = nextFrameRequest.textureFrame;
graph.PushInput(nextFrame).AssertOk();
graph.RenderOutput(webCamScreenController, nextFrame);
}
}