public static string[] FindModels(string filter) { string[] assets = AssetDatabase.FindAssets("t:ASMontrealLanguageModel " + filter); for (int s = 0; s < assets.Length; s++) { ASMontrealLanguageModel model = AssetDatabase.LoadAssetAtPath <ASMontrealLanguageModel>(AssetDatabase.GUIDToAssetPath(assets[s])); assets[s] = model.language; } return(assets); }
public static ASMontrealLanguageModel Load(int index) { string[] languageModelGUIDs = AssetDatabase.FindAssets("t:ASMontrealLanguageModel"); var settings = LipSyncEditorExtensions.GetProjectFile(); if (settings == null) { return(null); } if (settings.phonemeSet == null) { return(null); } if (languageModelGUIDs.Length > 0 && languageModelGUIDs.Length > index) { ASMontrealLanguageModel model = AssetDatabase.LoadAssetAtPath <ASMontrealLanguageModel>(AssetDatabase.GUIDToAssetPath(languageModelGUIDs[index])); if (model != null) { if (model.mappingMode == AutoSyncPhonemeMap.MappingMode.InternalMap && !string.IsNullOrEmpty(model.recommendedPhonemeSet) && model.recommendedPhonemeSet != settings.phonemeSet.scriptingName) { if (!EditorUtility.DisplayDialog("Wrong Phoneme Set", "Warning: You are using the '" + settings.phonemeSet.scriptingName + "' Phoneme Set, and this language model is designed for use with '" + model.recommendedPhonemeSet + "'. This may not provide usable results, are you sure you want to continue?", "Yes", "No")) { return(null); } } return(model); } } else { Debug.LogError("LipSync: Invalid Montreal language model index provided."); } return(null); }
public override void Process(LipSyncData inputClip, AutoSync.ASProcessDelegate callback) { string mfaPath = EditorPrefs.GetString("as_montrealfa_application_path"); string audioPath = AssetDatabase.GetAssetPath(inputClip.clip).Substring("/Assets".Length); if (audioPath == null) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path could not be found.", ClipFeatures.None)); return; } if (!AutoSyncUtility.VerifyProgramAtPath(mfaPath, "align")) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Montreal Forced Aligner application path is not verified.", ClipFeatures.None)); return; } // Get absolute path audioPath = Application.dataPath + "/" + audioPath; // Check Path if (audioPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0 || Path.GetFileNameWithoutExtension(audioPath).IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path contains invalid characters.", ClipFeatures.None)); return; } // Load Language Model ASMontrealLanguageModel model = ASMontrealLanguageModel.Load(languageModel); if (model == null) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model failed to load.", ClipFeatures.None)); return; } string basePath = model.GetBasePath(); string lexiconPath = ""; if (model.usePredefinedLexicon) { lexiconPath = basePath + model.lexiconPath; } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Support for generated lexicons using a G2P model is coming soon.", ClipFeatures.None)); return; } string adjustedName = Path.GetFileNameWithoutExtension(audioPath).Replace('.', '_').Replace(' ', '_').Replace('\\', '_').Replace('/', '_'); string corpusPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Corpus"; string outputPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Output"; // Delete folders if they already exist try { if (Directory.Exists(corpusPath)) { Directory.Delete(corpusPath, true); } if (Directory.Exists(outputPath)) { Directory.Delete(outputPath, true); } } catch { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Attempt to clear temporary MFA folders failed. Are they open in another application?", ClipFeatures.None)); return; } // Create temporary folders Directory.CreateDirectory(corpusPath); Directory.CreateDirectory(outputPath); // Copy or convert audio clip to corpus folder if (AutoSyncConversionUtility.IsConversionAvailable && useAudioConversion) { AutoSyncConversionUtility.StartConversion(audioPath, corpusPath + "/" + adjustedName + ".wav", AutoSyncConversionUtility.AudioFormat.WavPCM, 16000, 16, 1); } else { File.Copy(audioPath, corpusPath + "/" + adjustedName + Path.GetExtension(audioPath)); } // Create transcript file in corpus folder StreamWriter transcriptWriter = File.CreateText(corpusPath + "/" + adjustedName + ".lab"); transcriptWriter.Write(inputClip.transcript.Replace('-', ' ')); transcriptWriter.Close(); // Run aligner Directory.SetCurrentDirectory(Application.dataPath.Remove(Application.dataPath.Length - 6)); mfaPath = Path.GetFullPath(mfaPath); System.Diagnostics.Process process = new System.Diagnostics.Process(); process.StartInfo.FileName = mfaPath; #if UNITY_EDITOR_WIN process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\" --quiet"; #elif UNITY_EDITOR_OSX process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\""; #endif process.StartInfo.UseShellExecute = true; process.StartInfo.CreateNoWindow = true; process.Start(); process.WaitForExit(15000); if (!process.HasExited) { process.Kill(); process.Close(); } var outputFiles = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories); string textGridPath = ""; List <string> oovs = new List <string>(); bool nothingFound = true; for (int i = 0; i < outputFiles.Length; i++) { if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".textgrid") { textGridPath = outputFiles[i]; nothingFound = false; } else if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".txt") { string name = Path.GetFileNameWithoutExtension(outputFiles[i]); var reader = new StreamReader(outputFiles[i]); if (name == "oovs_found") { while (!reader.EndOfStream) { oovs.Add(reader.ReadLine()); } } reader.Close(); } } // Detect out-of-vocab words, filter and retry if enabled. if (oovs.Count > 0) { Debug.Log("Found out-of-vocabulary words:"); for (int i = 0; i < oovs.Count; i++) { Debug.Log(oovs[i]); } } if (nothingFound) { if (autoRetry) { if (attempts < maxAttempts - 1) { attempts++; Process(inputClip, callback); return; } } callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "MFA Application Failed. Check your audio encoding or enable conversion.", ClipFeatures.None)); return; } // Load in TextGrid TextGridUtility.TextGridItem[] items = TextGridUtility.ParseTextGridFile(textGridPath); var settings = LipSyncEditorExtensions.GetProjectFile(); bool needsMapping = true; Dictionary <string, string> phonemeMapper = null; if (model.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName) { needsMapping = false; } else { needsMapping = true; switch (model.mappingMode) { case AutoSyncPhonemeMap.MappingMode.InternalMap: phonemeMapper = model.phonemeMap.GenerateAtoBDictionary(); break; case AutoSyncPhonemeMap.MappingMode.ExternalMap: if (model.externalMap) { phonemeMapper = model.externalMap.phonemeMap.GenerateAtoBDictionary(); } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model specifies an external phoneme map, but no phoneme map was provided.", ClipFeatures.None)); return; } break; default: case AutoSyncPhonemeMap.MappingMode.AutoDetect: phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName); if (phonemeMapper == null) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, string.Format("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName), ClipFeatures.None)); return; } break; } if (phonemeMapper.Count == 0) { Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated."); } } // Get Phones List <PhonemeMarker> data = new List <PhonemeMarker>(); if (items != null && items.Length == 2) { for (int i = 0; i < items[1].intervals.Length; i++) { if (items[1].intervals[i] == null) { Debug.LogFormat("Interval {0} is null :o", i); continue; } string label = items[1].intervals[i].text.Split('"')[1]; label = System.Text.RegularExpressions.Regex.Replace(label, "[0-9]", ""); if (label != "sil") { if (phonemeMapper.ContainsKey(label)) { string phonemeName = needsMapping ? phonemeMapper[label] : label; bool found = false; int phoneme; for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++) { if (settings.phonemeSet.phonemes[phoneme].name == phonemeName) { found = true; break; } } if (found) { double start = items[1].intervals[i].xmin / inputClip.length; double end = items[1].intervals[i].xmax / inputClip.length; double length = end - start; if ((length * inputClip.length) < minLengthForSustain) { data.Add(new PhonemeMarker(phoneme, (float)(start + (length / 2)))); } else { data.Add(new PhonemeMarker(phoneme, (float)start, 1, true)); data.Add(new PhonemeMarker(phoneme, (float)end)); } } else { Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry."); } } else { Debug.LogWarning("Phoneme mapper does not contain '" + label + "' Skipping this entry."); } } } } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Data loaded from MFA TextGrid file was invalid or incomplete.", ClipFeatures.None)); return; } inputClip.phonemeData = data.ToArray(); if (oovs.Count > 0) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "Completed, but some words were not found. Check the console.", GetOutputCompatibility())); } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "", GetOutputCompatibility())); } return; }
private void OnEnable() { languageModelNames = ASMontrealLanguageModel.FindModels(); }