public static void ShowWindow(LipSync component, AnimatorController controller)
{
GestureSetupWizard window = EditorWindow.GetWindow <GestureSetupWizard>(true);
window.component = component;
window.controller = controller;
window.topMessage = "Setting up Gestures for " + controller.name + ".";
window.totalSteps = 2;
window.Focus();
window.titleContent = new GUIContent("Gesture Setup Wizard");
window.settings = LipSyncEditorExtensions.GetProjectFile();
}
public static ASMontrealLanguageModel Load(int index)
{
string[] languageModelGUIDs = AssetDatabase.FindAssets("t:ASMontrealLanguageModel");
var settings = LipSyncEditorExtensions.GetProjectFile();
if (settings == null)
{
return(null);
}
if (settings.phonemeSet == null)
{
return(null);
}
if (languageModelGUIDs.Length > 0 && languageModelGUIDs.Length > index)
{
ASMontrealLanguageModel model = AssetDatabase.LoadAssetAtPath <ASMontrealLanguageModel>(AssetDatabase.GUIDToAssetPath(languageModelGUIDs[index]));
if (model != null)
{
if (model.mappingMode == AutoSyncPhonemeMap.MappingMode.InternalMap && !string.IsNullOrEmpty(model.recommendedPhonemeSet) && model.recommendedPhonemeSet != settings.phonemeSet.scriptingName)
{
if (!EditorUtility.DisplayDialog("Wrong Phoneme Set", "Warning: You are using the '" + settings.phonemeSet.scriptingName + "' Phoneme Set, and this language model is designed for use with '" + model.recommendedPhonemeSet + "'. This may not provide usable results, are you sure you want to continue?", "Yes", "No"))
{
return(null);
}
}
return(model);
}
}
else
{
Debug.LogError("LipSync: Invalid Montreal language model index provided.");
}
return(null);
}
private List <PhonemeMarker> ParseOutput(string[] lines, ASPocketSphinxLanguageModel lm, AudioClip clip)
{
List <PhonemeMarker> results = new List <PhonemeMarker>();
Dictionary <string, string> phonemeMapper = null;
var settings = LipSyncEditorExtensions.GetProjectFile();
bool needsMapping = true;
if (lm.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName)
{
needsMapping = false;
}
else
{
needsMapping = true;
switch (lm.mappingMode)
{
case AutoSyncPhonemeMap.MappingMode.InternalMap:
phonemeMapper = lm.phonemeMap.GenerateAtoBDictionary();
break;
case AutoSyncPhonemeMap.MappingMode.ExternalMap:
if (lm.externalMap)
{
phonemeMapper = lm.externalMap.phonemeMap.GenerateAtoBDictionary();
}
else
{
Debug.LogError("Language Model specifies an external phoneme map, but no phoneme map was provided.");
return(null);
}
break;
default:
case AutoSyncPhonemeMap.MappingMode.AutoDetect:
phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(lm.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName);
if (phonemeMapper == null)
{
Debug.LogErrorFormat("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", lm.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName);
return(null);
}
break;
}
if (phonemeMapper.Count == 0)
{
Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated.");
}
}
NumberStyles style = NumberStyles.Number;
CultureInfo culture = CultureInfo.InvariantCulture;
foreach (string line in lines)
{
if (string.IsNullOrEmpty(line))
{
break;
}
string[] tokens = line.Split(' ');
try
{
if (tokens[0] != "SIL")
{
string phonemeName = needsMapping ? phonemeMapper[tokens[0]] : tokens[0];
float startTime = -1;
float.TryParse(tokens[1], style, culture, out startTime);
if (startTime > -1)
{
startTime /= clip.length;
}
else
{
Debug.LogWarning("Phoneme mapper returned invalid timestamp. Skipping this entry.");
continue;
}
bool found = false;
int phoneme;
for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++)
{
if (settings.phonemeSet.phonemes[phoneme].name == phonemeName)
{
found = true;
break;
}
}
if (found)
{
results.Add(new PhonemeMarker(phoneme, startTime));
}
else
{
Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry.");
}
}
}
catch (ArgumentOutOfRangeException)
{
Debug.LogWarning("Phoneme Label missing from return data. Skipping this entry.");
}
catch (KeyNotFoundException)
{
Debug.LogWarning("Phoneme Label '" + tokens[0] + "' not found in phoneme mapper. Skipping this entry.");
}
}
return(results);
}
private void FinishedProcessingMulti(LipSyncData outputData, AutoSync.ASProcessDelegateData data)
{
if (data.success)
{
var settings = LipSyncEditorExtensions.GetProjectFile();
// Create File
string outputPath = AssetDatabase.GetAssetPath(outputData.clip);
outputPath = Path.ChangeExtension(outputPath, xmlMode ? "xml" : "asset");
try
{
LipSyncClipSetup.SaveFile(settings, outputPath, xmlMode, outputData.transcript, outputData.length, outputData.phonemeData, outputData.emotionData,
outputData.gestureData, outputData.clip);
}
catch (Exception e)
{
Debug.LogError(e.StackTrace);
}
}
else
{
batchIncomplete = true;
string clipName = "Undefined";
if (outputData.clip)
{
clipName = outputData.clip.name;
}
Debug.LogErrorFormat("AutoSync: Processing failed on clip '{0}'. Continuing with batch.", clipName);
}
if (currentClip < clips.Count)
{
currentClip++;
if (autoSyncInstance == null)
{
autoSyncInstance = new AutoSync();
}
LipSyncData tempData = CreateInstance <LipSyncData>();
tempData.clip = clips[currentClip];
tempData.length = tempData.clip.length;
if (loadTranscripts)
{
tempData.transcript = AutoSyncUtility.TryGetTranscript(tempData.clip);
}
autoSyncInstance.RunSequence(currentModules.ToArray(), FinishedProcessingMulti, tempData);
}
else
{
AssetDatabase.Refresh();
EditorUtility.ClearProgressBar();
if (!batchIncomplete)
{
setup.ShowNotification(new GUIContent("Batch AutoSync Completed Successfully"));
}
else
{
setup.ShowNotification(new GUIContent("Batch AutoSync Completed With Errors"));
}
Close();
}
}
public override void Process(LipSyncData inputClip, AutoSync.ASProcessDelegate callback)
{
string mfaPath = EditorPrefs.GetString("as_montrealfa_application_path");
string audioPath = AssetDatabase.GetAssetPath(inputClip.clip).Substring("/Assets".Length);
if (audioPath == null)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path could not be found.", ClipFeatures.None));
return;
}
if (!AutoSyncUtility.VerifyProgramAtPath(mfaPath, "align"))
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Montreal Forced Aligner application path is not verified.", ClipFeatures.None));
return;
}
// Get absolute path
audioPath = Application.dataPath + "/" + audioPath;
// Check Path
if (audioPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0 || Path.GetFileNameWithoutExtension(audioPath).IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path contains invalid characters.", ClipFeatures.None));
return;
}
// Load Language Model
ASMontrealLanguageModel model = ASMontrealLanguageModel.Load(languageModel);
if (model == null)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model failed to load.", ClipFeatures.None));
return;
}
string basePath = model.GetBasePath();
string lexiconPath = "";
if (model.usePredefinedLexicon)
{
lexiconPath = basePath + model.lexiconPath;
}
else
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Support for generated lexicons using a G2P model is coming soon.", ClipFeatures.None));
return;
}
string adjustedName = Path.GetFileNameWithoutExtension(audioPath).Replace('.', '_').Replace(' ', '_').Replace('\\', '_').Replace('/', '_');
string corpusPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Corpus";
string outputPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Output";
// Delete folders if they already exist
try
{
if (Directory.Exists(corpusPath))
{
Directory.Delete(corpusPath, true);
}
if (Directory.Exists(outputPath))
{
Directory.Delete(outputPath, true);
}
}
catch
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Attempt to clear temporary MFA folders failed. Are they open in another application?", ClipFeatures.None));
return;
}
// Create temporary folders
Directory.CreateDirectory(corpusPath);
Directory.CreateDirectory(outputPath);
// Copy or convert audio clip to corpus folder
if (AutoSyncConversionUtility.IsConversionAvailable && useAudioConversion)
{
AutoSyncConversionUtility.StartConversion(audioPath, corpusPath + "/" + adjustedName + ".wav", AutoSyncConversionUtility.AudioFormat.WavPCM, 16000, 16, 1);
}
else
{
File.Copy(audioPath, corpusPath + "/" + adjustedName + Path.GetExtension(audioPath));
}
// Create transcript file in corpus folder
StreamWriter transcriptWriter = File.CreateText(corpusPath + "/" + adjustedName + ".lab");
transcriptWriter.Write(inputClip.transcript.Replace('-', ' '));
transcriptWriter.Close();
// Run aligner
Directory.SetCurrentDirectory(Application.dataPath.Remove(Application.dataPath.Length - 6));
mfaPath = Path.GetFullPath(mfaPath);
System.Diagnostics.Process process = new System.Diagnostics.Process();
process.StartInfo.FileName = mfaPath;
#if UNITY_EDITOR_WIN
process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\" --quiet";
#elif UNITY_EDITOR_OSX
process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\"";
#endif
process.StartInfo.UseShellExecute = true;
process.StartInfo.CreateNoWindow = true;
process.Start();
process.WaitForExit(15000);
if (!process.HasExited)
{
process.Kill();
process.Close();
}
var outputFiles = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories);
string textGridPath = "";
List <string> oovs = new List <string>();
bool nothingFound = true;
for (int i = 0; i < outputFiles.Length; i++)
{
if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".textgrid")
{
textGridPath = outputFiles[i];
nothingFound = false;
}
else if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".txt")
{
string name = Path.GetFileNameWithoutExtension(outputFiles[i]);
var reader = new StreamReader(outputFiles[i]);
if (name == "oovs_found")
{
while (!reader.EndOfStream)
{
oovs.Add(reader.ReadLine());
}
}
reader.Close();
}
}
// Detect out-of-vocab words, filter and retry if enabled.
if (oovs.Count > 0)
{
Debug.Log("Found out-of-vocabulary words:");
for (int i = 0; i < oovs.Count; i++)
{
Debug.Log(oovs[i]);
}
}
if (nothingFound)
{
if (autoRetry)
{
if (attempts < maxAttempts - 1)
{
attempts++;
Process(inputClip, callback);
return;
}
}
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "MFA Application Failed. Check your audio encoding or enable conversion.", ClipFeatures.None));
return;
}
// Load in TextGrid
TextGridUtility.TextGridItem[] items = TextGridUtility.ParseTextGridFile(textGridPath);
var settings = LipSyncEditorExtensions.GetProjectFile();
bool needsMapping = true;
Dictionary <string, string> phonemeMapper = null;
if (model.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName)
{
needsMapping = false;
}
else
{
needsMapping = true;
switch (model.mappingMode)
{
case AutoSyncPhonemeMap.MappingMode.InternalMap:
phonemeMapper = model.phonemeMap.GenerateAtoBDictionary();
break;
case AutoSyncPhonemeMap.MappingMode.ExternalMap:
if (model.externalMap)
{
phonemeMapper = model.externalMap.phonemeMap.GenerateAtoBDictionary();
}
else
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model specifies an external phoneme map, but no phoneme map was provided.", ClipFeatures.None));
return;
}
break;
default:
case AutoSyncPhonemeMap.MappingMode.AutoDetect:
phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName);
if (phonemeMapper == null)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, string.Format("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName), ClipFeatures.None));
return;
}
break;
}
if (phonemeMapper.Count == 0)
{
Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated.");
}
}
// Get Phones
List <PhonemeMarker> data = new List <PhonemeMarker>();
if (items != null && items.Length == 2)
{
for (int i = 0; i < items[1].intervals.Length; i++)
{
if (items[1].intervals[i] == null)
{
Debug.LogFormat("Interval {0} is null :o", i);
continue;
}
string label = items[1].intervals[i].text.Split('"')[1];
label = System.Text.RegularExpressions.Regex.Replace(label, "[0-9]", "");
if (label != "sil")
{
if (phonemeMapper.ContainsKey(label))
{
string phonemeName = needsMapping ? phonemeMapper[label] : label;
bool found = false;
int phoneme;
for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++)
{
if (settings.phonemeSet.phonemes[phoneme].name == phonemeName)
{
found = true;
break;
}
}
if (found)
{
double start = items[1].intervals[i].xmin / inputClip.length;
double end = items[1].intervals[i].xmax / inputClip.length;
double length = end - start;
if ((length * inputClip.length) < minLengthForSustain)
{
data.Add(new PhonemeMarker(phoneme, (float)(start + (length / 2))));
}
else
{
data.Add(new PhonemeMarker(phoneme, (float)start, 1, true));
data.Add(new PhonemeMarker(phoneme, (float)end));
}
}
else
{
Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry.");
}
}
else
{
Debug.LogWarning("Phoneme mapper does not contain '" + label + "' Skipping this entry.");
}
}
}
}
else
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Data loaded from MFA TextGrid file was invalid or incomplete.", ClipFeatures.None));
return;
}
inputClip.phonemeData = data.ToArray();
if (oovs.Count > 0)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "Completed, but some words were not found. Check the console.", GetOutputCompatibility()));
}
else
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "", GetOutputCompatibility()));
}
return;
}
public static void ShowWindow()
{
var settings = LipSyncEditorExtensions.GetProjectFile();
Selection.activeObject = settings;
}