void DrawFormant(ref FormantPair formant, string name, bool isDefaultAsset)
{
var f1 = EditorGUILayout.Slider($"{name} - F1", formant.f1, 0f, 4000f);
if (f1 != formant.f1 && !isDefaultAsset)
{
Undo.RecordObject(target, $"Changed {name} F1");
if (f1 > formant.f2)
{
f1 = formant.f2;
}
formant.f1 = f1;
}
var f2 = EditorGUILayout.Slider($"{name} - F2", formant.f2, 0f, 4000f);
if (f2 != formant.f2 && !isDefaultAsset)
{
Undo.RecordObject(target, $"Changed {name} F2");
if (f2 < formant.f1)
{
f2 = formant.f1;
}
formant.f2 = f2;
}
}
void InvokeCallback()
{
if (onLipSyncUpdate == null)
{
return;
}
var vowelInfo = config.checkThirdFormant ?
LipSyncUtil.GetVowel(jobResult_[0].f1, jobResult_[0].f2, jobResult_[0].f3, profile) :
LipSyncUtil.GetVowel(new FormantPair(jobResult_[0].f1, jobResult_[0].f2), profile);
float volume = jobResult_[0].volume;
FormantPair formant = vowelInfo.formant;
Vowel vowel = vowelInfo.vowel;
if (config.checkSecondDerivative)
{
var vowelInfoBySecondDerivative = config.checkThirdFormant ?
LipSyncUtil.GetVowel(jobResult_[1].f1, jobResult_[1].f2, jobResult_[1].f3, profile) :
LipSyncUtil.GetVowel(new FormantPair(jobResult_[1].f1, jobResult_[1].f2), profile);
if (vowelInfo.diff > vowelInfoBySecondDerivative.diff)
{
formant = vowelInfoBySecondDerivative.formant;
vowel = vowelInfoBySecondDerivative.vowel;
}
}
UpdateLipSyncInfo(volume, formant, vowel);
onLipSyncUpdate.Invoke(result);
}
void UpdateLipSyncInfo(float volume, FormantPair formant, Vowel vowel)
{
float sf = 1f - openSmoothness;
float sb = 1f - closeSmoothness;
float preVolume = result.volume;
rawResult_.volume = volume;
float normalizedVolume = Mathf.Clamp((volume - minVolume) / (maxVolume - minVolume), 0f, 1f);
float smooth = normalizedVolume > preVolume ? sf : sb;
result.volume += (normalizedVolume - preVolume) * smooth;
rawResult_.formant = result.formant = formant;
if (volume < minVolume)
{
return;
}
if (vowel == Vowel.None)
{
return;
}
float max = 0f;
float sum = 0f;
for (int i = (int)Vowel.A; i <= (int)Vowel.None; ++i)
{
var key = (Vowel)i;
float target = key == vowel ? 1f : 0f;
float value = rawResult_.vowels[key];
value += (target - value) * (1f - vowelTransitionSmoothness);
if (value > max)
{
rawResult_.mainVowel = key;
max = value;
}
rawResult_.vowels[key] = value;
sum += value;
}
result.mainVowel = rawResult_.mainVowel;
for (int i = (int)Vowel.A; i <= (int)Vowel.None; ++i)
{
var key = (Vowel)i;
if (sum > Mathf.Epsilon)
{
result.vowels[key] = rawResult_.vowels[key] / sum;
}
else
{
result.vowels[key] = 0f;
}
}
}
public static VowelInfo GetVowel(FormantPair formant, Profile profile)
{
var info = new VowelInfo();
info.vowel = Vowel.None;
info.formant = formant;
float diffA = FormantPair.Dist(formant, profile.formantA);
float diffI = FormantPair.Dist(formant, profile.formantI);
float diffU = FormantPair.Dist(formant, profile.formantU);
float diffE = FormantPair.Dist(formant, profile.formantE);
float diffO = FormantPair.Dist(formant, profile.formantO);
float minDiff = math.min(diffA, math.min(diffI, math.min(diffU, math.min(diffE, diffO))));
info.diff = minDiff;
if (!profile.useErrorRange || minDiff < profile.maxErrorRange)
{
if (diffA == minDiff)
{
info.vowel = Vowel.A;
}
else if (diffI == minDiff)
{
info.vowel = Vowel.I;
}
else if (diffU == minDiff)
{
info.vowel = Vowel.U;
}
else if (diffE == minDiff)
{
info.vowel = Vowel.E;
}
else if (diffO == minDiff)
{
info.vowel = Vowel.O;
}
}
return(info);
}
public static float Dist(FormantPair a, FormantPair b)
{
return(Mathf.Sqrt(Mathf.Pow(a.f1 - b.f1, 2f) + Mathf.Pow(a.f2 - b.f2, 2f)));
}
public static void DrawFormants(Profile profile, LipSyncInfo result = null)
{
var origColor = Handles.color;
var area = GUILayoutUtility.GetRect(Screen.width, 300f);
area = EditorGUI.IndentedRect(area);
var margin = new Margin(10, 10f, 30f, 40f);
var range = new Vector2(1200f, 4000f);
DrawGrid(
area,
Color.white,
new Color(1f, 1f, 1f, 0.5f),
margin,
range,
new Vector2(6f, 4f));
if (!profile)
{
return;
}
float xMin = area.x + margin.left;
float xMax = area.xMax - margin.right;
float yMin = area.y + margin.top;
float yMax = area.yMax - margin.bottom;
float width = xMax - xMin;
float height = yMax - yMin;
var colors = new Color[]
{
new Color(1f, 0f, 0f, 1f),
new Color(0f, 1f, 0f, 1f),
new Color(0f, 0f, 1f, 1f),
new Color(1f, 1f, 0f, 1f),
new Color(0f, 1f, 1f, 1f),
};
var formants = new FormantPair[]
{
profile.formantA,
profile.formantI,
profile.formantU,
profile.formantE,
profile.formantO,
};
var vowelLabels = new string[]
{
"A",
"I",
"U",
"E",
"O",
};
float dx = width / range.x;
float dy = height / range.y;
for (int i = 0; i < formants.Length; ++i)
{
var f = formants[i];
float x = xMin + dx * f.f1;
float y = yMin + (height - dy * f.f2);
float rx = profile.maxErrorRange * dx;
float ry = profile.maxErrorRange * dy;
var center = new Vector3(x, y, 0f);
var color = colors[i];
Handles.color = color;
Handles.DrawSolidDisc(center, Vector3.forward, 5f);
float factor = result != null ? result.vowels[(Vowel)i] : 0f;
color.a = Mathf.Lerp(0.15f, 0.5f, factor);
Handles.color = color;
DrawEllipse(center, rx, ry, new Rect(xMin, yMin, width, height));
EditorGUI.LabelField(new Rect(x + 5f, y - 20f, 50f, 20f), vowelLabels[i]);
}
if (result != null)
{
float x = xMin + result.formant.f1 * dx;
float y = yMin + (height - result.formant.f2 * dy);
float size = Mathf.Lerp(2f, 20f, Mathf.Min(result.volume / 0.1f, 1f));
var center = new Vector3(x, y, 0f);
Handles.color = Color.white;
Handles.DrawWireDisc(center, Vector3.forward, size);
}
Handles.color = origColor;
}