public static Vector2 GetRetinalDiameter(this HeadsetModel headset)
{
return(new Vector2(
headset.GetRetinalDiameter(Axis.Horizontal),
headset.GetRetinalDiameter(Axis.Vertical)
));
}
public static float GetFieldOfView(this HeadsetModel headset, Axis axis)
{
float vertical;
switch (headset)
{
case HeadsetModel.Fove: vertical = 95; break;
case HeadsetModel.VivePro: vertical = 110; break;
case HeadsetModel.None60: vertical = 60; break;
case HeadsetModel.None120: vertical = 120; break;
case HeadsetModel.FHD60: vertical = 60; break;
default: throw new Exception();
}
switch (axis)
{
case Axis.Horizontal: return(AuxMath.HorizontalFoV(vertical, headset.GetAspectRatio()));
case Axis.Vertical: return(vertical);
default: throw new Exception();
}
}
/// <summary>
/// Pixel (i, j) -> Visual Angle (um)
/// </summary>
public static Vector2 PixelToRetina(int i, int j, HeadsetModel headset)
{
var horizontalDiameter = headset.GetRetinalDiameter(Axis.Horizontal);
var verticalDiameter = headset.GetRetinalDiameter(Axis.Vertical);
var x = ((float)i / headset.GetWidth() - .5f) * horizontalDiameter;
var y = ((float)j / headset.GetHeight() - .5f) * verticalDiameter;
return(new Vector2(x, y));
}
public static Texture2D CreateTexture(this HeadsetModel headset)
{
var texture = new Texture2D(headset.GetWidth(), headset.GetHeight(), TextureFormat.RGBAFloat, false, true)
{
anisoLevel = 0,
filterMode = FilterMode.Point
};
return(texture);
}
/// <summary>
/// Retinal position (um) -> Pixel (i, j)
/// </summary>
public static Vector2 RetinaToPixel(Vector2 retina, HeadsetModel headset)
{
var horizontalDiameter = headset.GetRetinalDiameter(Axis.Horizontal);
var verticalDiameter = headset.GetRetinalDiameter(Axis.Vertical);
var x = (retina.x / horizontalDiameter + .5f) * headset.GetWidth();
var y = (retina.y / verticalDiameter + .5f) * headset.GetHeight();
return(new Vector2(x, y));
}
public void RenderBitmaps()
{
var visual = new DrawingVisual();
using (var c = visual.RenderOpen())
{
c.PushClip(new RectangleGeometry(_rect));
c.DrawRectangle(new SolidColorBrush(Color.FromArgb(0, 0, 0, 0)), null, _rect);
if (MouseModel != null)
{
MouseModel.CloseDrawingContext();
c.DrawRectangle(new VisualBrush(MouseModel), new Pen(), MouseModel.RelativeRect);
}
if (HeadsetModel != null)
{
HeadsetModel.CloseDrawingContext();
c.DrawRectangle(new VisualBrush(HeadsetModel), new Pen(), HeadsetModel.RelativeRect);
}
if (GenericModel != null)
{
GenericModel.CloseDrawingContext();
c.DrawRectangle(new VisualBrush(GenericModel), new Pen(), GenericModel.RelativeRect);
}
if (MousematModel != null)
{
MousematModel.CloseDrawingContext();
c.DrawRectangle(new VisualBrush(MousematModel), new Pen(), MousematModel.RelativeRect);
}
KeyboardModel.CloseDrawingContext();
c.DrawRectangle(new VisualBrush(KeyboardModel), new Pen(), KeyboardModel.RelativeRect);
c.Pop();
}
var frameBitmap = ImageUtilities.DrawingVisualToBitmap(visual, _rect);
if (MouseModel != null)
{
MouseBitmap = MouseModel.GetBitmapFromFrame(frameBitmap);
}
if (HeadsetModel != null)
{
HeadsetBitmap = HeadsetModel.GetBitmapFromFrame(frameBitmap);
}
if (GenericModel != null)
{
GenericBitmap = GenericModel.GetBitmapFromFrame(frameBitmap);
}
if (MousematModel != null)
{
MousematBitmap = MousematModel.GetBitmapFromFrame(frameBitmap);
}
KeyboardBitmap = KeyboardModel.GetBitmapFromFrame(frameBitmap);
}
public bool IsReady(HeadsetModel headset)
{
var ready = true;
foreach (var pattern in Enumerate <ElectrodePattern>())
{
ready &= IsPhospheneTextureReady(headset, pattern);
}
return(ready && IsAxonTextureReady(headset));
}
void OnEnable()
{
_dataType = DataType.Phosphene;
_headset = HeadsetModel.VivePro;
_pattern = ElectrodePattern.POLYRETINA;
_layout = ElectrodeLayout._80x150;
_path = "Assets/";
_gpuAccel = true;
cpuStarted = false;
}
public bool IsPhospheneTextureReady(HeadsetModel headset, ElectrodePattern pattern)
{
var ready = true;
foreach (var layout in Enumerate <ElectrodeLayout>())
{
ready &= IsPhospheneTextureReady(headset, pattern, layout);
}
return(ready);
}
/*
* Public methods
*/
/// <summary>
/// Creates a data texture. A threadGroup is returned to keep track of creation progress; Null if GPU accelerated.
/// </summary>
public static ThreadGroup Create(DataType type, HeadsetModel headset, ElectrodePattern pattern, ElectrodeLayout layout, bool gpuAccel, string path)
{
if (gpuAccel)
{
StartGPU(type, headset, pattern, layout, path);
return(null);
}
else
{
return(StartCPU(type, headset, pattern, layout, path));
}
}
private int GetAxonTextureIndex(HeadsetModel headset)
{
var numPatterns = Count <ElectrodePattern>();
var numLayouts = Count <ElectrodeLayout>();
var headsetIndex = headset.GetIndex();
var headsetOffset = headsetIndex * (numPatterns * numLayouts + 1);
var axonOffset = numPatterns * numLayouts;
return(headsetOffset + axonOffset);
}
void OnGUI()
{
UnityGUI.Header("Coordinates");
headset = UnityGUI.Enum("Headset", headset);
UnityGUI.Space();
UnityGUI.BeginHorizontal();
UnityGUI.Label(" ", "X", UnityGUI.BoldLabel);
UnityGUI.Label("Y", UnityGUI.BoldLabel);
UnityGUI.EndHorizontal();
UnityGUI.BeginHorizontal();
UnityGUI.Label("Resolution", headset.GetWidth().ToString());
UnityGUI.Label(headset.GetHeight().ToString());
UnityGUI.EndHorizontal();
UnityGUI.BeginHorizontal();
UnityGUI.Label("Field of View", ((int)headset.GetFieldOfView(Axis.Horizontal)).ToString());
UnityGUI.Label(((int)headset.GetFieldOfView(Axis.Vertical)).ToString());
UnityGUI.EndHorizontal();
UnityGUI.BeginHorizontal();
UnityGUI.Label("Diameter (um)", ((int)headset.GetRetinalDiameter(Axis.Horizontal)).ToString());
UnityGUI.Label(((int)headset.GetRetinalDiameter(Axis.Vertical)).ToString());
UnityGUI.EndHorizontal();
UnityGUI.Space();
Draw("Pixel", ref pixel);
Draw("Visual Angle", ref angle);
Draw("Retina (um)", ref retina);
Draw("Polar", ref polar);
UnityGUI.Separator();
UnityGUI.Header("Electrode Count");
UnityGUI.BeginChangeCheck();
pattern = UnityGUI.Enum("Pattern", pattern);
layout = UnityGUI.Enum("Layout", layout);
fieldOfView = UnityGUI.Float("Field of View", fieldOfView);
var changed = UnityGUI.EndChangeCheck();
if (changed)
{
numElectrodes = pattern.GetElectrodePositions(layout, fieldOfView).Length;
}
UnityGUI.Space();
UnityGUI.Label("Electode Count", numElectrodes.ToString());
}
public static RenderTexture CreateRenderTexture(this HeadsetModel headset)
{
var texture = new RenderTexture(headset.GetWidth(), headset.GetHeight(), 0, RenderTextureFormat.ARGBFloat, RenderTextureReadWrite.Linear)
{
anisoLevel = 0,
antiAliasing = 1,
enableRandomWrite = true,
filterMode = FilterMode.Point
};
texture.Create();
return(texture);
}
/*
* Private methods
*/
private int GetPhospheneTextureIndex(HeadsetModel headset, ElectrodePattern pattern, ElectrodeLayout layout)
{
var numPatterns = Count <ElectrodePattern>();
var numLayouts = Count <ElectrodeLayout>();
var headsetIndex = headset.GetIndex();
var patternIndex = pattern.GetIndex();
var layoutIndex = layout.GetIndex();
var headsetOffset = headsetIndex * (numPatterns * numLayouts + 1);
var patternOffset = patternIndex * numLayouts;
return(headsetOffset + patternOffset + layoutIndex);
}
/*
* Public methods
*/
public static AxonPoint[,] CalculateMatrix(HeadsetModel headset)
{
var width = headset.GetWidth();
var height = headset.GetHeight();
var matrix = new AxonPoint[width, height];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
matrix[i, j] = CalculatePoint(i, j, headset);
}
}
return(matrix);
}
public static Vector2 GetResolution(this HeadsetModel headset)
{
switch (headset)
{
case HeadsetModel.Fove: return(new Vector2(1792, 2016));
case HeadsetModel.VivePro: return(new Vector2(2036, 2260));
case HeadsetModel.None60: return(new Vector2(1000, 1000));
case HeadsetModel.None120: return(new Vector2(1000, 1000));
case HeadsetModel.FHD60: return(new Vector2(1920, 1080));
default: throw new Exception();
}
}
public static Vector2[,] CalculateMatrix(HeadsetModel headset, Vector2[] electrodes)
{
var width = headset.GetWidth();
var height = headset.GetHeight();
var matrix = new Vector2[width, height];
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
matrix[i, j] = CalculatePoint(i, j, headset, electrodes);
}
}
return(matrix);
}
public static int GetRefreshRate(this HeadsetModel headset)
{
switch (headset)
{
case HeadsetModel.Fove: return(70);
case HeadsetModel.VivePro: return(XRDevice.isPresent ? -1 : 90);
case HeadsetModel.None60: return(100);
case HeadsetModel.None120: return(100);
case HeadsetModel.FHD60: return(100);
default: throw new Exception();
}
}
void OnGUI()
{
UnityGUI.Space();
_dataType = UnityGUI.Enum("Data Type", _dataType);
UnityGUI.Space();
UnityGUI.Label("Parameters", UnityGUI.BoldLabel);
_headset = UnityGUI.Enum("Headset Model", _headset);
if (_dataType == DataType.Phosphene)
{
_pattern = UnityGUI.Enum("Electrode Pattern", _pattern);
_layout = UnityGUI.Enum("Electrode Layout", _layout);
}
UnityGUI.Space();
_gpuAccel = UnityGUI.ToggleLeft("GPU Accelerated", _gpuAccel);
if (UnityGUI.Button(!cpuStarted ? "Start" : "Stop", new GUIOptions {
maxWidth = 50
}))
{
if (!cpuStarted)
{
Start();
}
else
{
StopCPU();
}
}
if (threadGroup != null)
{
UnityGUI.Enabled = cpuStarted || !_gpuAccel;
UnityGUI.Space();
UnityGUI.Label("CPU Threads", UnityGUI.BoldLabel);
for (int i = 0; i < threadGroup.NumThreads; i++)
{
UnityGUI.Label("Thread " + i.ToString(), (threadGroup.Progress[i] * 100).ToString("N0") + "%");
}
}
}
private static ThreadGroup StartCPU(DataType type, HeadsetModel headset, ElectrodePattern pattern, ElectrodeLayout layout, string path)
{
// declare colour matrix and electrodes (if needed)
var matrix = new Color[headset.GetWidth(), headset.GetHeight()];
var electrodes = type == DataType.Phosphene ? pattern.GetElectrodePositions(layout)
: default;
// setup threadGroup
var threadGroup = new ThreadGroup();
threadGroup.OnAllThreadsFinished += () =>
{
EditorCallback.InvokeOnMainThread(() =>
{
var asset = headset.CreateTexture();
asset.SetPixels(matrix.Flatten(false, true, true));
asset.Apply();
SaveAsset(asset, path);
});
};
// generate the texture asynchronously
threadGroup.ProcessArray(headset.GetWidth(), (i) =>
{
var n = headset.GetHeight();
for (int j = 0; j < n; j++)
{
switch (type)
{
case DataType.Phosphene:
matrix[i, j] = PhospheneMatrix.CalculatePoint(i, j, headset, electrodes).ToColour();
matrix[i, j] = AddRandomSeeds(matrix[i, j]);
break;
case DataType.Axon:
matrix[i, j] = AxonMatrix.CalculatePoint(i, j, headset).Colour;
break;
}
}
});
// provide caller with threadGroup for completion statistics
return(threadGroup);
}
/*
* Protected methods
*/
protected void Initialise(string phosShader, string tailShader)
{
// load shaders
phos = new Material(Shader.Find(phosShader));
tail = new Material(Shader.Find(tailShader));
if (phos == null || tail == null)
{
Debug.LogError($"{name} does not have a material.");
return;
}
// upload electrode/axon textures to the GPU
phos.SetTexture(SP.electrodeTexture, epiretinalData.GetPhospheneTexture(headset, pattern, layout));
tail.SetTexture(SP.axonTexture, epiretinalData.GetAxonTexture(headset));
// create texture for the fading data
fadeRT = new DoubleBufferedRenderTexture(headset.GetWidth(), headset.GetHeight());
fadeRT.Initialise(new Color(1, 0, 0, 0));
// overrides
if (overrideCameraFOV)
{
Prosthesis.Instance.Camera.fieldOfView = headset.GetFieldOfView(Axis.Vertical);
}
if (overrideRefreshRate)
{
Application.targetFrameRate = headset.GetRefreshRate();
}
// cache texture-related variables
lastHeadset = headset;
lastPattern = pattern;
lastLayout = layout;
// initialise eye gaze tracking
EyeGaze.Initialise(eyeGazeSource, headset);
// initialise camera target eye
UpdateCameraTargetEye();
}
public static AxonPoint CalculatePoint(int x, int y, HeadsetModel headset)
{
var polar = CoordinateSystem.PixelToPolar(x, y, headset);
var bestPhi0 = ClosestWholePhi0(polar);
for (int i = 0; i < 5; i++)
{
var deviation = .55f / Mathf.Pow(10, i);
bestPhi0 = ClosestPhi0(polar, bestPhi0 - deviation, bestPhi0 + deviation, 11);
}
return(new AxonPoint()
{
phi0 = bestPhi0,
rho = polar.x,
b = AxonModel.CalculateB(bestPhi0),
c = AxonModel.CalculateC(bestPhi0)
});
}
/*
* Private methods
*/
private static void StartGPU(DataType type, HeadsetModel headset, ElectrodePattern pattern, ElectrodeLayout layout, string path)
{
var shader = LoadAsset <ComputeShader>($"{type}");
var kernel = shader.FindKernel("CSMain");
var electrodeBuffer = default(ComputeBuffer);
if (type == DataType.Phosphene)
{
var electrodes = pattern.GetElectrodePositions(layout);
electrodeBuffer = new ComputeBuffer(electrodes.Length, sizeof(float) * 2);
electrodeBuffer.SetData(electrodes);
shader.SetBuffer(kernel, "_electrodes", electrodeBuffer);
}
var texture = headset.CreateRenderTexture();
shader.SetTexture(kernel, "_result", texture);
shader.SetVector("_headset_diameter", headset.GetRetinalDiameter());
shader.SetVector("_headset_resolution", headset.GetResolution());
shader.Dispatch(kernel, headset.GetWidth() / 8, headset.GetHeight() / 8, 1);
electrodeBuffer?.Dispose();
Texture2D asset = texture.ToTexture2D(TextureFormat.RGBAFloat, true);
texture.Release();
asset.anisoLevel = 0;
asset.filterMode = FilterMode.Point;
if (type == DataType.Phosphene)
{
AddRandomSeeds(asset);
}
SaveAsset(asset, path);
}
public static int GetHeight(this HeadsetModel headset)
{
return((int)headset.GetResolution().y);
}
public static int GetWidth(this HeadsetModel headset)
{
return((int)headset.GetResolution().x);
}
public void SetAxonTexture(HeadsetModel headset, Texture2D asset)
{
data[GetAxonTextureIndex(headset)] = asset;
}
public bool IsPhospheneTextureReady(HeadsetModel headset, ElectrodePattern pattern, ElectrodeLayout layout)
{
return(GetPhospheneTexture(headset, pattern, layout) != null);
}
public bool IsAxonTextureReady(HeadsetModel headset)
{
return(GetAxonTexture(headset) != null);
}
public static float GetAspectRatio(this HeadsetModel headset)
{
return((float)headset.GetWidth() / headset.GetHeight());
}
public static float GetRetinalDiameter(this HeadsetModel headset, Axis axis)
{
return(CoordinateSystem.FovToRetinalDiameter(headset.GetFieldOfView(axis)));
}
