void DrawFormant(ref FormantPair formant, string name, bool isDefaultAsset) { var f1 = EditorGUILayout.Slider($"{name} - F1", formant.f1, 0f, 4000f); if (f1 != formant.f1 && !isDefaultAsset) { Undo.RecordObject(target, $"Changed {name} F1"); if (f1 > formant.f2) { f1 = formant.f2; } formant.f1 = f1; } var f2 = EditorGUILayout.Slider($"{name} - F2", formant.f2, 0f, 4000f); if (f2 != formant.f2 && !isDefaultAsset) { Undo.RecordObject(target, $"Changed {name} F2"); if (f2 < formant.f1) { f2 = formant.f1; } formant.f2 = f2; } }
void InvokeCallback() { if (onLipSyncUpdate == null) { return; } var vowelInfo = config.checkThirdFormant ? LipSyncUtil.GetVowel(jobResult_[0].f1, jobResult_[0].f2, jobResult_[0].f3, profile) : LipSyncUtil.GetVowel(new FormantPair(jobResult_[0].f1, jobResult_[0].f2), profile); float volume = jobResult_[0].volume; FormantPair formant = vowelInfo.formant; Vowel vowel = vowelInfo.vowel; if (config.checkSecondDerivative) { var vowelInfoBySecondDerivative = config.checkThirdFormant ? LipSyncUtil.GetVowel(jobResult_[1].f1, jobResult_[1].f2, jobResult_[1].f3, profile) : LipSyncUtil.GetVowel(new FormantPair(jobResult_[1].f1, jobResult_[1].f2), profile); if (vowelInfo.diff > vowelInfoBySecondDerivative.diff) { formant = vowelInfoBySecondDerivative.formant; vowel = vowelInfoBySecondDerivative.vowel; } } UpdateLipSyncInfo(volume, formant, vowel); onLipSyncUpdate.Invoke(result); }
void UpdateLipSyncInfo(float volume, FormantPair formant, Vowel vowel) { float sf = 1f - openSmoothness; float sb = 1f - closeSmoothness; float preVolume = result.volume; rawResult_.volume = volume; float normalizedVolume = Mathf.Clamp((volume - minVolume) / (maxVolume - minVolume), 0f, 1f); float smooth = normalizedVolume > preVolume ? sf : sb; result.volume += (normalizedVolume - preVolume) * smooth; rawResult_.formant = result.formant = formant; if (volume < minVolume) { return; } if (vowel == Vowel.None) { return; } float max = 0f; float sum = 0f; for (int i = (int)Vowel.A; i <= (int)Vowel.None; ++i) { var key = (Vowel)i; float target = key == vowel ? 1f : 0f; float value = rawResult_.vowels[key]; value += (target - value) * (1f - vowelTransitionSmoothness); if (value > max) { rawResult_.mainVowel = key; max = value; } rawResult_.vowels[key] = value; sum += value; } result.mainVowel = rawResult_.mainVowel; for (int i = (int)Vowel.A; i <= (int)Vowel.None; ++i) { var key = (Vowel)i; if (sum > Mathf.Epsilon) { result.vowels[key] = rawResult_.vowels[key] / sum; } else { result.vowels[key] = 0f; } } }
public static VowelInfo GetVowel(FormantPair formant, Profile profile) { var info = new VowelInfo(); info.vowel = Vowel.None; info.formant = formant; float diffA = FormantPair.Dist(formant, profile.formantA); float diffI = FormantPair.Dist(formant, profile.formantI); float diffU = FormantPair.Dist(formant, profile.formantU); float diffE = FormantPair.Dist(formant, profile.formantE); float diffO = FormantPair.Dist(formant, profile.formantO); float minDiff = math.min(diffA, math.min(diffI, math.min(diffU, math.min(diffE, diffO)))); info.diff = minDiff; if (!profile.useErrorRange || minDiff < profile.maxErrorRange) { if (diffA == minDiff) { info.vowel = Vowel.A; } else if (diffI == minDiff) { info.vowel = Vowel.I; } else if (diffU == minDiff) { info.vowel = Vowel.U; } else if (diffE == minDiff) { info.vowel = Vowel.E; } else if (diffO == minDiff) { info.vowel = Vowel.O; } } return(info); }
public static float Dist(FormantPair a, FormantPair b) { return(Mathf.Sqrt(Mathf.Pow(a.f1 - b.f1, 2f) + Mathf.Pow(a.f2 - b.f2, 2f))); }
public static void DrawFormants(Profile profile, LipSyncInfo result = null) { var origColor = Handles.color; var area = GUILayoutUtility.GetRect(Screen.width, 300f); area = EditorGUI.IndentedRect(area); var margin = new Margin(10, 10f, 30f, 40f); var range = new Vector2(1200f, 4000f); DrawGrid( area, Color.white, new Color(1f, 1f, 1f, 0.5f), margin, range, new Vector2(6f, 4f)); if (!profile) { return; } float xMin = area.x + margin.left; float xMax = area.xMax - margin.right; float yMin = area.y + margin.top; float yMax = area.yMax - margin.bottom; float width = xMax - xMin; float height = yMax - yMin; var colors = new Color[] { new Color(1f, 0f, 0f, 1f), new Color(0f, 1f, 0f, 1f), new Color(0f, 0f, 1f, 1f), new Color(1f, 1f, 0f, 1f), new Color(0f, 1f, 1f, 1f), }; var formants = new FormantPair[] { profile.formantA, profile.formantI, profile.formantU, profile.formantE, profile.formantO, }; var vowelLabels = new string[] { "A", "I", "U", "E", "O", }; float dx = width / range.x; float dy = height / range.y; for (int i = 0; i < formants.Length; ++i) { var f = formants[i]; float x = xMin + dx * f.f1; float y = yMin + (height - dy * f.f2); float rx = profile.maxErrorRange * dx; float ry = profile.maxErrorRange * dy; var center = new Vector3(x, y, 0f); var color = colors[i]; Handles.color = color; Handles.DrawSolidDisc(center, Vector3.forward, 5f); float factor = result != null ? result.vowels[(Vowel)i] : 0f; color.a = Mathf.Lerp(0.15f, 0.5f, factor); Handles.color = color; DrawEllipse(center, rx, ry, new Rect(xMin, yMin, width, height)); EditorGUI.LabelField(new Rect(x + 5f, y - 20f, 50f, 20f), vowelLabels[i]); } if (result != null) { float x = xMin + result.formant.f1 * dx; float y = yMin + (height - result.formant.f2 * dy); float size = Mathf.Lerp(2f, 20f, Mathf.Min(result.volume / 0.1f, 1f)); var center = new Vector3(x, y, 0f); Handles.color = Color.white; Handles.DrawWireDisc(center, Vector3.forward, size); } Handles.color = origColor; }