private Texture textureForInputKey(string key, bool segmentationMap) { Texture2D texture = null; if (inputData[key] is Texture) { texture = inputData[key] as Texture2D; } else if (inputData[key] is GameObject) { GameObject go = ((GameObject)inputData[key]); Camera mainCamera = go.GetComponent <Camera>(); if (segmentationMap) { ImageSynthesis synthesis = go.GetComponent <ImageSynthesis>(); if (synthesis == null) { synthesis = go.AddComponent <ImageSynthesis>(); } Camera cam = synthesis.capturePasses[2].camera; texture = RunwayUtils.CameraToTexture(cam, mainCamera.pixelWidth, mainCamera.pixelHeight); } else { texture = RunwayUtils.CameraToTexture(mainCamera, mainCamera.pixelWidth, mainCamera.pixelHeight); } } if (texture == null) { return(null); } return(texture); }
void RenderModelOptions() { GUILayout.Space(10); GUILayout.Label("SETUP OPTIONS", sectionTitleStyle); GUILayout.Space(5); GUILayout.BeginHorizontal("box"); GUILayout.BeginVertical(); Field[] options = getSelectedModel().options == null ? new Field[0] : getSelectedModel().options; for (var i = 0; i < options.Length; i++) { GUILayout.Space(5); Field option = options[i]; if (option.type == "category" || option.type == "file") { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.Label(RunwayUtils.FormatFieldName(option.name)); GUILayout.FlexibleSpace(); optionSelectionIndices[i] = EditorGUILayout.Popup(optionSelectionIndices[i], option.oneOf); GUILayout.EndHorizontal(); } } GUILayout.Space(5); GUILayout.EndVertical(); GUILayout.EndHorizontal(); }
public static float[] RandomVector(int length, float mean, float sigma) { float[] vec = new float[length]; for (int i = 0; i < length; i++) { vec[i] = RunwayUtils.GenerateNormalRandom(mean, sigma); } return vec; }
void RunInference() { Field[] inputs = getSelectedModel().commands[0].inputs; Field[] outputs = getSelectedModel().commands[0].outputs; Dictionary <string, object> dataToSend = new Dictionary <string, object>(); for (var i = 0; i < inputs.Length; i++) { Field input = inputs[i]; object value = inputData[input.name]; if (input.type.Equals("image")) { Texture2D tex = textureForInputKey(input.name, false) as Texture2D; byte[] data = RunwayUtils.TextureToPNG(tex); dataToSend[input.name] = "data:image/png;base64," + System.Convert.ToBase64String(data); } else if (input.type.Equals("segmentation")) { Texture2D tex = textureForInputKey(input.name, true) as Texture2D; byte[] data = RunwayUtils.TextureToPNG(tex); dataToSend[input.name] = "data:image/png;base64," + System.Convert.ToBase64String(data); } else if (input.type.Equals("vector")) { dataToSend[input.name] = RunwayUtils.RandomVector(input.length, input.samplingMean, input.samplingStd); } else { dataToSend[input.name] = value; } } this.StartCoroutine(RunwayHub.runInference(runningSession.url, getFilteredModels()[selectedModelIndex].commands[0].name, dataToSend, (outputData) => { this.isProcessingInput = false; if (outputData == null) { EditorUtility.DisplayDialog("Inference Error", "There was an error processing this input", "OK"); return; } for (var i = 0; i < outputs.Length; i++) { object value = outputData[outputs[i].name]; if (outputs[i].type.Equals("image")) { string stringValue = value as string; int dataStartIndex = stringValue.IndexOf("base64,") + 7; byte[] outputImg = System.Convert.FromBase64String(((string)value).Substring(dataStartIndex)); Texture2D tex = new Texture2D(2, 2); // Once image is loaded, texture will auto-resize tex.LoadImage(outputImg); this.lastOutput = tex; } } Repaint(); })); }
void RenderObjectTagger(Field input, int index) { GUILayout.BeginHorizontal(); GUILayout.FlexibleSpace(); GUILayout.BeginVertical(); GUILayout.BeginHorizontal("GroupBox", GUILayout.MaxWidth(300)); GUILayout.BeginVertical(); foreach (string label in input.labels) { GameObject[] objs = RunwayUtils.GetObjectsWithLabelTag(label); if (objs.Length > 0) { GUILayout.Label(System.String.Format("{0} objects labeled as {1}", objs.Length, label)); } } GUILayout.Space(10); GUILayout.BeginHorizontal(); GUILayout.Label("Select Label:"); selectedLabel = RunwayUtils.Dropdown(selectedLabel, input.labels); GUILayout.EndHorizontal(); using (new EditorGUI.DisabledScope(Selection.gameObjects.Length == 0)) { if (GUILayout.Button(System.String.Format("Label selected objects with {0} ({1})", selectedLabel, Selection.gameObjects.Length))) { RunwayUtils.AddTag(selectedLabel); foreach (GameObject go in Selection.gameObjects) { go.tag = selectedLabel; } } if (GUILayout.Button(System.String.Format("Unlabel selected objects ({0})", Selection.gameObjects.Length))) { RunwayUtils.AddTag(selectedLabel); foreach (GameObject go in Selection.gameObjects) { go.tag = "Untagged"; } } } GUILayout.EndVertical(); GUILayout.EndHorizontal(); GUILayout.EndHorizontal(); GUILayout.FlexibleSpace(); GUILayout.EndVertical(); }
void RenderInputsAndOutputs() { Field[] inputs = getFilteredModels()[selectedModelIndex].commands[0].inputs; Field[] outputs = getFilteredModels()[selectedModelIndex].commands[0].outputs; GUILayout.Space(10); GUILayout.Label("INPUT", sectionTitleStyle); GUILayout.Space(5); GUILayout.BeginHorizontal("box"); GUILayout.BeginVertical(); for (int i = 0; i < inputs.Length; i++) { GUILayout.Space(5); Field input = inputs[i]; GUILayout.BeginVertical(); if (input.type.Equals("image") || input.type.Equals("segmentation")) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.FlexibleSpace(); GUILayout.Label(RunwayUtils.FormatFieldName(input.name), boldTextStyle); GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); GUILayout.Space(5); } if (input.type.Equals("image")) { RenderImageInput(input, i); } else if (input.type.Equals("segmentation")) { RenderSegmentationInput(input, i); } else if (input.type.Equals("text")) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name))); GUILayout.FlexibleSpace(); inputData[input.name] = EditorGUILayout.TextField(inputData[input.name] as string, GUILayout.MaxWidth(250)); GUILayout.EndHorizontal(); } else if (input.type.Equals("category")) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name))); GUILayout.FlexibleSpace(); inputData[input.name] = RunwayUtils.Dropdown(inputData[input.name] as string, input.oneOf); GUILayout.EndHorizontal(); } else if (input.type.Equals("number")) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name))); GUILayout.FlexibleSpace(); if (RunwayUtils.IsAnInteger(input.step)) { if (input.hasMin && input.hasMax && input.hasStep) { int value = inputData[input.name] is int?(int)inputData[input.name] : (int)Convert.ToSingle(input.defaultValue); inputData[input.name] = EditorGUILayout.IntSlider(value, (int)input.min, (int)input.max); } else { int value = inputData[input.name] is int?(int)inputData[input.name] : (int)Convert.ToSingle(input.defaultValue); inputData[input.name] = EditorGUILayout.IntField(value); } } else { if (input.hasMin && input.hasMax && input.hasStep) { float value = inputData[input.name] is float?(float)inputData[input.name] : (float)Convert.ToSingle(input.defaultValue); inputData[input.name] = EditorGUILayout.Slider(value, input.min, input.max); } else { float value = inputData[input.name] is float?(float)inputData[input.name] : Convert.ToSingle(input.defaultValue); inputData[input.name] = EditorGUILayout.FloatField(value); } } GUILayout.EndHorizontal(); } else if (input.type.Equals("boolean")) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.Label(System.String.Format("{0}:", RunwayUtils.FormatFieldName(input.name))); bool value = inputData[input.name] is bool?(bool)inputData[input.name] : false; GUILayout.FlexibleSpace(); inputData[input.name] = EditorGUILayout.Toggle(value, GUILayout.Width(20)); GUILayout.EndHorizontal(); } GUILayout.Space(5); GUILayout.EndVertical(); } GUILayout.EndVertical(); GUILayout.EndHorizontal(); GUILayout.Space(10); GUILayout.Label("OUTPUT", sectionTitleStyle); GUILayout.Space(5); GUILayout.BeginHorizontal("box"); GUILayout.BeginVertical(); GUILayout.Space(5); GUILayout.BeginHorizontal(horizontalStyle); GUILayout.FlexibleSpace(); if (this.lastOutput) { RenderTextureInfo(this.lastOutput); } else { RenderNotAvailable(); } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); GUILayout.BeginHorizontal(); GUILayout.FlexibleSpace(); if (GUILayout.Button("Preview")) { outputWindow = GetWindow <RunwayOutputWindow>(false, "Runway - Model Output", true); } GUILayout.Space(5); if (this.lastOutput && GUILayout.Button("Save")) { string path = EditorUtility.SaveFilePanel("Save as PNG", "", "ModelOutput.png", "png"); byte[] data = RunwayUtils.TextureToPNG(this.lastOutput); File.WriteAllBytes(path, data); } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); GUILayout.Space(5); GUILayout.EndVertical(); GUILayout.EndHorizontal(); if (lastOutput != null && outputWindow != null) { outputWindow.texture = lastOutput; outputWindow.Repaint(); } }
void RenderSegmentationInput(Field input, int index) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.FlexibleSpace(); Texture tex = textureForInputKey(input.name, true); if (inputData[input.name] != null) { RenderTextureInfo(tex); } else { RenderNotAvailable(); } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); GUILayout.Space(5); GUILayout.BeginHorizontal(); GUILayout.FlexibleSpace(); if (userPickedObjectForIndex == index) { GameObject go = EditorGUIUtility.GetObjectPickerObject() as GameObject; inputData[input.name] = go; if (go != null) { ImageSynthesis synthesis = go.GetComponent <ImageSynthesis>(); synthesis.labels = input.labels; synthesis.colors = input.colors; synthesis.defaultColor = input.defaultColor; inputWidths[index] = maxWidths[index] = (int)go.GetComponent <Camera>().pixelWidth; inputHeights[index] = maxHeights[index] = (int)go.GetComponent <Camera>().pixelHeight; } userPickedObjectForIndex = -1; } if (Event.current.commandName == "ObjectSelectorUpdated" && EditorGUIUtility.GetObjectPickerControlID() == index) { userPickedObjectForIndex = index; } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); GUILayout.BeginHorizontal(); GUILayout.FlexibleSpace(); if (GUILayout.Button("Select")) { EditorGUIUtility.ShowObjectPicker <UnityEngine.Object>(inputData[input.name] as UnityEngine.Object, true, "t:Camera", index); } GUILayout.Space(5); if (GUILayout.Button("Preview")) { if (index == 0) { inputWindows[index] = GetWindow <RunwayInput1Window>(false, "Runway - Model Input 1", true); } else { inputWindows[index] = GetWindow <RunwayInput2Window>(false, "Runway - Model Input 2", true); } } GUILayout.Space(5); if (GUILayout.Button("Save")) { string path = EditorUtility.SaveFilePanel("Save as PNG", "", "ModelInput.png", "png"); byte[] data = RunwayUtils.TextureToPNG(tex as Texture2D); File.WriteAllBytes(path, data); } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); RenderObjectTagger(input, index); if (inputData[input.name] != null) { if (inputWindows.ContainsKey(index)) { inputWindows[index].texture = tex; inputWindows[index].Repaint(); } } }
void RenderImageInput(Field input, int index) { GUILayout.BeginHorizontal(horizontalStyle); GUILayout.FlexibleSpace(); Texture tex = textureForInputKey(input.name, false); if (tex != null) { RenderTextureInfo(tex); } else { RenderNotAvailable(); } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); GUILayout.Space(5); GUILayout.BeginHorizontal(); GUILayout.FlexibleSpace(); if (GUILayout.Button("Select")) { EditorGUIUtility.ShowObjectPicker <UnityEngine.Object>(inputData[input.name] as UnityEngine.Object, true, "t:Texture t:Camera", index); } if (userPickedObjectForIndex == index) { inputData[input.name] = EditorGUIUtility.GetObjectPickerObject(); userPickedObjectForIndex = -1; } if (Event.current.commandName == "ObjectSelectorUpdated" && EditorGUIUtility.GetObjectPickerControlID() == index) { userPickedObjectForIndex = index; } GUILayout.Space(5); if (GUILayout.Button("Preview")) { if (index == 0) { inputWindows[index] = GetWindow <RunwayInput1Window>(false, "Runway - Model Input 1", true); } else { inputWindows[index] = GetWindow <RunwayInput2Window>(false, "Runway - Model Input 2", true); } } GUILayout.Space(5); if (GUILayout.Button("Save")) { string path = EditorUtility.SaveFilePanel("Save as PNG", "", "ModelInput.png", "png"); byte[] data = RunwayUtils.TextureToPNG(tex as Texture2D); File.WriteAllBytes(path, data); } GUILayout.FlexibleSpace(); GUILayout.EndHorizontal(); if (inputData[input.name] != null) { if (inputWindows.ContainsKey(index)) { inputWindows[index].texture = tex; inputWindows[index].Repaint(); } } }