private Texture textureForInputKey(string key, bool segmentationMap)
{
Texture2D texture = null;
if (inputData[key] is Texture)
{
texture = inputData[key] as Texture2D;
}
else if (inputData[key] is GameObject)
{
GameObject go = ((GameObject)inputData[key]);
Camera mainCamera = go.GetComponent <Camera>();
if (segmentationMap)
{
ImageSynthesis synthesis = go.GetComponent <ImageSynthesis>();
if (synthesis == null)
{
synthesis = go.AddComponent <ImageSynthesis>();
}
Camera cam = synthesis.capturePasses[2].camera;
texture = RunwayUtils.CameraToTexture(cam, mainCamera.pixelWidth, mainCamera.pixelHeight);
}
else
{
texture = RunwayUtils.CameraToTexture(mainCamera, mainCamera.pixelWidth, mainCamera.pixelHeight);
}
}
if (texture == null)
{
return(null);
}
return(texture);
}
void RenderModelOptions()
{
GUILayout.Space(10);
GUILayout.Label("SETUP OPTIONS", sectionTitleStyle);
GUILayout.Space(5);
GUILayout.BeginHorizontal("box");
GUILayout.BeginVertical();
Field[] options = getSelectedModel().options == null ? new Field[0] : getSelectedModel().options;
for (var i = 0; i < options.Length; i++)
{
GUILayout.Space(5);
Field option = options[i];
if (option.type == "category" || option.type == "file")
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.Label(RunwayUtils.FormatFieldName(option.name));
GUILayout.FlexibleSpace();
optionSelectionIndices[i] = EditorGUILayout.Popup(optionSelectionIndices[i], option.oneOf);
GUILayout.EndHorizontal();
}
}
GUILayout.Space(5);
GUILayout.EndVertical();
GUILayout.EndHorizontal();
}
public static float[] RandomVector(int length, float mean, float sigma)
{
float[] vec = new float[length];
for (int i = 0; i < length; i++)
{
vec[i] = RunwayUtils.GenerateNormalRandom(mean, sigma);
}
return vec;
}
void RunInference()
{
Field[] inputs = getSelectedModel().commands[0].inputs;
Field[] outputs = getSelectedModel().commands[0].outputs;
Dictionary <string, object> dataToSend = new Dictionary <string, object>();
for (var i = 0; i < inputs.Length; i++)
{
Field input = inputs[i];
object value = inputData[input.name];
if (input.type.Equals("image"))
{
Texture2D tex = textureForInputKey(input.name, false) as Texture2D;
byte[] data = RunwayUtils.TextureToPNG(tex);
dataToSend[input.name] = "data:image/png;base64," + System.Convert.ToBase64String(data);
}
else if (input.type.Equals("segmentation"))
{
Texture2D tex = textureForInputKey(input.name, true) as Texture2D;
byte[] data = RunwayUtils.TextureToPNG(tex);
dataToSend[input.name] = "data:image/png;base64," + System.Convert.ToBase64String(data);
}
else if (input.type.Equals("vector"))
{
dataToSend[input.name] = RunwayUtils.RandomVector(input.length, input.samplingMean, input.samplingStd);
}
else
{
dataToSend[input.name] = value;
}
}
this.StartCoroutine(RunwayHub.runInference(runningSession.url, getFilteredModels()[selectedModelIndex].commands[0].name, dataToSend, (outputData) =>
{
this.isProcessingInput = false;
if (outputData == null)
{
EditorUtility.DisplayDialog("Inference Error", "There was an error processing this input", "OK");
return;
}
for (var i = 0; i < outputs.Length; i++)
{
object value = outputData[outputs[i].name];
if (outputs[i].type.Equals("image"))
{
string stringValue = value as string;
int dataStartIndex = stringValue.IndexOf("base64,") + 7;
byte[] outputImg = System.Convert.FromBase64String(((string)value).Substring(dataStartIndex));
Texture2D tex = new Texture2D(2, 2); // Once image is loaded, texture will auto-resize
tex.LoadImage(outputImg);
this.lastOutput = tex;
}
}
Repaint();
}));
}
void RenderObjectTagger(Field input, int index)
{
GUILayout.BeginHorizontal();
GUILayout.FlexibleSpace();
GUILayout.BeginVertical();
GUILayout.BeginHorizontal("GroupBox", GUILayout.MaxWidth(300));
GUILayout.BeginVertical();
foreach (string label in input.labels)
{
GameObject[] objs = RunwayUtils.GetObjectsWithLabelTag(label);
if (objs.Length > 0)
{
GUILayout.Label(System.String.Format("{0} objects labeled as {1}", objs.Length, label));
}
}
GUILayout.Space(10);
GUILayout.BeginHorizontal();
GUILayout.Label("Select Label:");
selectedLabel = RunwayUtils.Dropdown(selectedLabel, input.labels);
GUILayout.EndHorizontal();
using (new EditorGUI.DisabledScope(Selection.gameObjects.Length == 0))
{
if (GUILayout.Button(System.String.Format("Label selected objects with {0} ({1})", selectedLabel, Selection.gameObjects.Length)))
{
RunwayUtils.AddTag(selectedLabel);
foreach (GameObject go in Selection.gameObjects)
{
go.tag = selectedLabel;
}
}
if (GUILayout.Button(System.String.Format("Unlabel selected objects ({0})", Selection.gameObjects.Length)))
{
RunwayUtils.AddTag(selectedLabel);
foreach (GameObject go in Selection.gameObjects)
{
go.tag = "Untagged";
}
}
}
GUILayout.EndVertical();
GUILayout.EndHorizontal();
GUILayout.EndHorizontal();
GUILayout.FlexibleSpace();
GUILayout.EndVertical();
}
void RenderInputsAndOutputs()
{
Field[] inputs = getFilteredModels()[selectedModelIndex].commands[0].inputs;
Field[] outputs = getFilteredModels()[selectedModelIndex].commands[0].outputs;
GUILayout.Space(10);
GUILayout.Label("INPUT", sectionTitleStyle);
GUILayout.Space(5);
GUILayout.BeginHorizontal("box");
GUILayout.BeginVertical();
for (int i = 0; i < inputs.Length; i++)
{
GUILayout.Space(5);
Field input = inputs[i];
GUILayout.BeginVertical();
if (input.type.Equals("image") || input.type.Equals("segmentation"))
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.FlexibleSpace();
GUILayout.Label(RunwayUtils.FormatFieldName(input.name), boldTextStyle);
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
GUILayout.Space(5);
}
if (input.type.Equals("image"))
{
RenderImageInput(input, i);
}
else if (input.type.Equals("segmentation"))
{
RenderSegmentationInput(input, i);
}
else if (input.type.Equals("text"))
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name)));
GUILayout.FlexibleSpace();
inputData[input.name] = EditorGUILayout.TextField(inputData[input.name] as string, GUILayout.MaxWidth(250));
GUILayout.EndHorizontal();
}
else if (input.type.Equals("category"))
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name)));
GUILayout.FlexibleSpace();
inputData[input.name] = RunwayUtils.Dropdown(inputData[input.name] as string, input.oneOf);
GUILayout.EndHorizontal();
}
else if (input.type.Equals("number"))
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name)));
GUILayout.FlexibleSpace();
if (RunwayUtils.IsAnInteger(input.step))
{
if (input.hasMin && input.hasMax && input.hasStep)
{
int value = inputData[input.name] is int?(int)inputData[input.name] : (int)Convert.ToSingle(input.defaultValue);
inputData[input.name] = EditorGUILayout.IntSlider(value, (int)input.min, (int)input.max);
}
else
{
int value = inputData[input.name] is int?(int)inputData[input.name] : (int)Convert.ToSingle(input.defaultValue);
inputData[input.name] = EditorGUILayout.IntField(value);
}
}
else
{
if (input.hasMin && input.hasMax && input.hasStep)
{
float value = inputData[input.name] is float?(float)inputData[input.name] : (float)Convert.ToSingle(input.defaultValue);
inputData[input.name] = EditorGUILayout.Slider(value, input.min, input.max);
}
else
{
float value = inputData[input.name] is float?(float)inputData[input.name] : Convert.ToSingle(input.defaultValue);
inputData[input.name] = EditorGUILayout.FloatField(value);
}
}
GUILayout.EndHorizontal();
}
else if (input.type.Equals("boolean"))
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name)));
bool value = inputData[input.name] is bool?(bool)inputData[input.name] : false;
GUILayout.FlexibleSpace();
inputData[input.name] = EditorGUILayout.Toggle(value, GUILayout.Width(20));
GUILayout.EndHorizontal();
}
GUILayout.Space(5);
GUILayout.EndVertical();
}
GUILayout.EndVertical();
GUILayout.EndHorizontal();
GUILayout.Space(10);
GUILayout.Label("OUTPUT", sectionTitleStyle);
GUILayout.Space(5);
GUILayout.BeginHorizontal("box");
GUILayout.BeginVertical();
GUILayout.Space(5);
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.FlexibleSpace();
if (this.lastOutput)
{
RenderTextureInfo(this.lastOutput);
}
else
{
RenderNotAvailable();
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
GUILayout.BeginHorizontal();
GUILayout.FlexibleSpace();
if (GUILayout.Button("Preview"))
{
outputWindow = GetWindow <RunwayOutputWindow>(false, "Runway - Model Output", true);
}
GUILayout.Space(5);
if (this.lastOutput && GUILayout.Button("Save"))
{
string path = EditorUtility.SaveFilePanel("Save as PNG", "", "ModelOutput.png", "png");
byte[] data = RunwayUtils.TextureToPNG(this.lastOutput);
File.WriteAllBytes(path, data);
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
GUILayout.Space(5);
GUILayout.EndVertical();
GUILayout.EndHorizontal();
if (lastOutput != null && outputWindow != null)
{
outputWindow.texture = lastOutput;
outputWindow.Repaint();
}
}
void RenderSegmentationInput(Field input, int index)
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.FlexibleSpace();
Texture tex = textureForInputKey(input.name, true);
if (inputData[input.name] != null)
{
RenderTextureInfo(tex);
}
else
{
RenderNotAvailable();
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
GUILayout.Space(5);
GUILayout.BeginHorizontal();
GUILayout.FlexibleSpace();
if (userPickedObjectForIndex == index)
{
GameObject go = EditorGUIUtility.GetObjectPickerObject() as GameObject;
inputData[input.name] = go;
if (go != null)
{
ImageSynthesis synthesis = go.GetComponent <ImageSynthesis>();
synthesis.labels = input.labels;
synthesis.colors = input.colors;
synthesis.defaultColor = input.defaultColor;
inputWidths[index] = maxWidths[index] = (int)go.GetComponent <Camera>().pixelWidth;
inputHeights[index] = maxHeights[index] = (int)go.GetComponent <Camera>().pixelHeight;
}
userPickedObjectForIndex = -1;
}
if (Event.current.commandName == "ObjectSelectorUpdated" && EditorGUIUtility.GetObjectPickerControlID() == index)
{
userPickedObjectForIndex = index;
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
GUILayout.BeginHorizontal();
GUILayout.FlexibleSpace();
if (GUILayout.Button("Select"))
{
EditorGUIUtility.ShowObjectPicker <UnityEngine.Object>(inputData[input.name] as UnityEngine.Object, true, "t:Camera", index);
}
GUILayout.Space(5);
if (GUILayout.Button("Preview"))
{
if (index == 0)
{
inputWindows[index] = GetWindow <RunwayInput1Window>(false, "Runway - Model Input 1", true);
}
else
{
inputWindows[index] = GetWindow <RunwayInput2Window>(false, "Runway - Model Input 2", true);
}
}
GUILayout.Space(5);
if (GUILayout.Button("Save"))
{
string path = EditorUtility.SaveFilePanel("Save as PNG", "", "ModelInput.png", "png");
byte[] data = RunwayUtils.TextureToPNG(tex as Texture2D);
File.WriteAllBytes(path, data);
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
RenderObjectTagger(input, index);
if (inputData[input.name] != null)
{
if (inputWindows.ContainsKey(index))
{
inputWindows[index].texture = tex;
inputWindows[index].Repaint();
}
}
}
void RenderImageInput(Field input, int index)
{
GUILayout.BeginHorizontal(horizontalStyle);
GUILayout.FlexibleSpace();
Texture tex = textureForInputKey(input.name, false);
if (tex != null)
{
RenderTextureInfo(tex);
}
else
{
RenderNotAvailable();
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
GUILayout.Space(5);
GUILayout.BeginHorizontal();
GUILayout.FlexibleSpace();
if (GUILayout.Button("Select"))
{
EditorGUIUtility.ShowObjectPicker <UnityEngine.Object>(inputData[input.name] as UnityEngine.Object, true, "t:Texture t:Camera", index);
}
if (userPickedObjectForIndex == index)
{
inputData[input.name] = EditorGUIUtility.GetObjectPickerObject();
userPickedObjectForIndex = -1;
}
if (Event.current.commandName == "ObjectSelectorUpdated" && EditorGUIUtility.GetObjectPickerControlID() == index)
{
userPickedObjectForIndex = index;
}
GUILayout.Space(5);
if (GUILayout.Button("Preview"))
{
if (index == 0)
{
inputWindows[index] = GetWindow <RunwayInput1Window>(false, "Runway - Model Input 1", true);
}
else
{
inputWindows[index] = GetWindow <RunwayInput2Window>(false, "Runway - Model Input 2", true);
}
}
GUILayout.Space(5);
if (GUILayout.Button("Save"))
{
string path = EditorUtility.SaveFilePanel("Save as PNG", "", "ModelInput.png", "png");
byte[] data = RunwayUtils.TextureToPNG(tex as Texture2D);
File.WriteAllBytes(path, data);
}
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
if (inputData[input.name] != null)
{
if (inputWindows.ContainsKey(index))
{
inputWindows[index].texture = tex;
inputWindows[index].Repaint();
}
}
}
