public void RunModuleSafely(AutoSyncModule module, LipSyncData data, ASProcessDelegate callback, bool silent = false) { if (finalData == null) { finalData = new ASProcessDelegateData(true, "", ClipFeatures.None); } if (AutoSyncUtility.CheckIsClipCompatible(data, module)) { if (!silent) { if (moduleSequence != null) { EditorUtility.DisplayProgressBar("AutoSync", string.Format("Processing module {0}/{1}. Please Wait.", index + 1, moduleSequence.Length), index + 1f / (float)moduleSequence.Length); } else { EditorUtility.DisplayProgressBar("AutoSync", "Processing module, Please Wait.", 1f); } } module.Process(data, callback); } else { EditorUtility.ClearProgressBar(); callback.Invoke(data, new ASProcessDelegateData(false, "Module failed compatibility check.", ClipFeatures.None)); moduleSequence = null; finalData = null; } }
public void RunModuleSafely(AutoSyncModule module, LipSyncData data, ASProcessDelegate callback, bool silent = false) { if (finalData == null) { finalData = new ASProcessDelegateData(true, "", ClipFeatures.None); } var missingFeatures = AutoSyncUtility.GetMissingClipFeatures(data, module); if (missingFeatures == ClipFeatures.None) { if (!silent) { if (moduleSequence != null) { EditorUtility.DisplayProgressBar("AutoSync", string.Format("Processing module {0}/{1}. Please Wait.", index + 1, moduleSequence.Length), index + 1f / (float)moduleSequence.Length); } else { EditorUtility.DisplayProgressBar("AutoSync", "Processing module, Please Wait.", 1f); } } module.Process(data, callback); } else { EditorUtility.ClearProgressBar(); callback.Invoke(data, new ASProcessDelegateData(false, string.Format("Failed: Missing {0}. See console for details.", missingFeatures.ToString()), ClipFeatures.None)); if ((missingFeatures & ClipFeatures.AudioClip) == ClipFeatures.AudioClip) { Debug.Log("Missing AudioClip. This module requires an audioclip in order to work. Make sure you've assigned an AudioClip at the top of the Clip Editor before running AutoSync."); } if ((missingFeatures & ClipFeatures.Emotions) == ClipFeatures.Emotions) { Debug.Log("Missing Emotions. This module requires Emotion markers in order to work. You will either need to add some using the Emotions tab of the Clip Editor, or by adding a module first that will create them."); } if ((missingFeatures & ClipFeatures.Gestures) == ClipFeatures.Gestures) { Debug.Log("Missing Gestures. This module requires Gesture markers in order to work. You will either need to add some using the Gestures tab of the Clip Editor, or by adding a module first that will create them."); } if ((missingFeatures & ClipFeatures.Phonemes) == ClipFeatures.Phonemes) { Debug.Log("Missing Phonemes. This module requires Phoneme markers in order to work. You will either need to add some using the Phonemes tab of the Clip Editor, or by adding a module first that will create them."); } if ((missingFeatures & ClipFeatures.Transcript) == ClipFeatures.Transcript) { Debug.Log("Missing Transcript. This module requires a transcript in order to work. The Clip Editor can load one from a text file with the same name as your AudioClip, or you can either add one from Edit>Clip Settings or add a module first that will create them."); } moduleSequence = null; finalData = null; } }
private List <PhonemeMarker> ParseOutput(string[] lines, ASPocketSphinxLanguageModel lm, AudioClip clip) { List <PhonemeMarker> results = new List <PhonemeMarker>(); Dictionary <string, string> phonemeMapper = null; var settings = LipSyncEditorExtensions.GetProjectFile(); bool needsMapping = true; if (lm.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName) { needsMapping = false; } else { needsMapping = true; switch (lm.mappingMode) { case AutoSyncPhonemeMap.MappingMode.InternalMap: phonemeMapper = lm.phonemeMap.GenerateAtoBDictionary(); break; case AutoSyncPhonemeMap.MappingMode.ExternalMap: if (lm.externalMap) { phonemeMapper = lm.externalMap.phonemeMap.GenerateAtoBDictionary(); } else { Debug.LogError("Language Model specifies an external phoneme map, but no phoneme map was provided."); return(null); } break; default: case AutoSyncPhonemeMap.MappingMode.AutoDetect: phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(lm.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName); if (phonemeMapper == null) { Debug.LogErrorFormat("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", lm.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName); return(null); } break; } if (phonemeMapper.Count == 0) { Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated."); } } NumberStyles style = NumberStyles.Number; CultureInfo culture = CultureInfo.InvariantCulture; foreach (string line in lines) { if (string.IsNullOrEmpty(line)) { break; } string[] tokens = line.Split(' '); try { if (tokens[0] != "SIL") { string phonemeName = needsMapping ? phonemeMapper[tokens[0]] : tokens[0]; float startTime = -1; float.TryParse(tokens[1], style, culture, out startTime); if (startTime > -1) { startTime /= clip.length; } else { Debug.LogWarning("Phoneme mapper returned invalid timestamp. Skipping this entry."); continue; } bool found = false; int phoneme; for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++) { if (settings.phonemeSet.phonemes[phoneme].name == phonemeName) { found = true; break; } } if (found) { results.Add(new PhonemeMarker(phoneme, startTime)); } else { Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry."); } } } catch (ArgumentOutOfRangeException) { Debug.LogWarning("Phoneme Label missing from return data. Skipping this entry."); } catch (KeyNotFoundException) { Debug.LogWarning("Phoneme Label '" + tokens[0] + "' not found in phoneme mapper. Skipping this entry."); } } return(results); }
public static bool CheckSoX() { string soXPath = EditorPrefs.GetString("LipSync_SoXPath"); return(AutoSyncUtility.VerifyProgramAtPath(soXPath, "SoX")); }
public override void Process(LipSyncData inputClip, AutoSync.ASProcessDelegate callback) { string mfaPath = EditorPrefs.GetString("as_montrealfa_application_path"); string audioPath = AssetDatabase.GetAssetPath(inputClip.clip).Substring("/Assets".Length); if (audioPath == null) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path could not be found.", ClipFeatures.None)); return; } if (!AutoSyncUtility.VerifyProgramAtPath(mfaPath, "align")) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Montreal Forced Aligner application path is not verified.", ClipFeatures.None)); return; } // Get absolute path audioPath = Application.dataPath + "/" + audioPath; // Check Path if (audioPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0 || Path.GetFileNameWithoutExtension(audioPath).IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path contains invalid characters.", ClipFeatures.None)); return; } // Load Language Model ASMontrealLanguageModel model = ASMontrealLanguageModel.Load(languageModel); if (model == null) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model failed to load.", ClipFeatures.None)); return; } string basePath = model.GetBasePath(); string lexiconPath = ""; if (model.usePredefinedLexicon) { lexiconPath = basePath + model.lexiconPath; } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Support for generated lexicons using a G2P model is coming soon.", ClipFeatures.None)); return; } string adjustedName = Path.GetFileNameWithoutExtension(audioPath).Replace('.', '_').Replace(' ', '_').Replace('\\', '_').Replace('/', '_'); string corpusPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Corpus"; string outputPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Output"; // Delete folders if they already exist try { if (Directory.Exists(corpusPath)) { Directory.Delete(corpusPath, true); } if (Directory.Exists(outputPath)) { Directory.Delete(outputPath, true); } } catch { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Attempt to clear temporary MFA folders failed. Are they open in another application?", ClipFeatures.None)); return; } // Create temporary folders Directory.CreateDirectory(corpusPath); Directory.CreateDirectory(outputPath); // Copy or convert audio clip to corpus folder if (AutoSyncConversionUtility.IsConversionAvailable && useAudioConversion) { AutoSyncConversionUtility.StartConversion(audioPath, corpusPath + "/" + adjustedName + ".wav", AutoSyncConversionUtility.AudioFormat.WavPCM, 16000, 16, 1); } else { File.Copy(audioPath, corpusPath + "/" + adjustedName + Path.GetExtension(audioPath)); } // Create transcript file in corpus folder StreamWriter transcriptWriter = File.CreateText(corpusPath + "/" + adjustedName + ".lab"); transcriptWriter.Write(inputClip.transcript.Replace('-', ' ')); transcriptWriter.Close(); // Run aligner Directory.SetCurrentDirectory(Application.dataPath.Remove(Application.dataPath.Length - 6)); mfaPath = Path.GetFullPath(mfaPath); System.Diagnostics.Process process = new System.Diagnostics.Process(); process.StartInfo.FileName = mfaPath; #if UNITY_EDITOR_WIN process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\" --quiet"; #elif UNITY_EDITOR_OSX process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\""; #endif process.StartInfo.UseShellExecute = true; process.StartInfo.CreateNoWindow = true; process.Start(); process.WaitForExit(15000); if (!process.HasExited) { process.Kill(); process.Close(); } var outputFiles = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories); string textGridPath = ""; List <string> oovs = new List <string>(); bool nothingFound = true; for (int i = 0; i < outputFiles.Length; i++) { if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".textgrid") { textGridPath = outputFiles[i]; nothingFound = false; } else if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".txt") { string name = Path.GetFileNameWithoutExtension(outputFiles[i]); var reader = new StreamReader(outputFiles[i]); if (name == "oovs_found") { while (!reader.EndOfStream) { oovs.Add(reader.ReadLine()); } } reader.Close(); } } // Detect out-of-vocab words, filter and retry if enabled. if (oovs.Count > 0) { Debug.Log("Found out-of-vocabulary words:"); for (int i = 0; i < oovs.Count; i++) { Debug.Log(oovs[i]); } } if (nothingFound) { if (autoRetry) { if (attempts < maxAttempts - 1) { attempts++; Process(inputClip, callback); return; } } callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "MFA Application Failed. Check your audio encoding or enable conversion.", ClipFeatures.None)); return; } // Load in TextGrid TextGridUtility.TextGridItem[] items = TextGridUtility.ParseTextGridFile(textGridPath); var settings = LipSyncEditorExtensions.GetProjectFile(); bool needsMapping = true; Dictionary <string, string> phonemeMapper = null; if (model.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName) { needsMapping = false; } else { needsMapping = true; switch (model.mappingMode) { case AutoSyncPhonemeMap.MappingMode.InternalMap: phonemeMapper = model.phonemeMap.GenerateAtoBDictionary(); break; case AutoSyncPhonemeMap.MappingMode.ExternalMap: if (model.externalMap) { phonemeMapper = model.externalMap.phonemeMap.GenerateAtoBDictionary(); } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model specifies an external phoneme map, but no phoneme map was provided.", ClipFeatures.None)); return; } break; default: case AutoSyncPhonemeMap.MappingMode.AutoDetect: phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName); if (phonemeMapper == null) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, string.Format("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName), ClipFeatures.None)); return; } break; } if (phonemeMapper.Count == 0) { Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated."); } } // Get Phones List <PhonemeMarker> data = new List <PhonemeMarker>(); if (items != null && items.Length == 2) { for (int i = 0; i < items[1].intervals.Length; i++) { if (items[1].intervals[i] == null) { Debug.LogFormat("Interval {0} is null :o", i); continue; } string label = items[1].intervals[i].text.Split('"')[1]; label = System.Text.RegularExpressions.Regex.Replace(label, "[0-9]", ""); if (label != "sil") { if (phonemeMapper.ContainsKey(label)) { string phonemeName = needsMapping ? phonemeMapper[label] : label; bool found = false; int phoneme; for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++) { if (settings.phonemeSet.phonemes[phoneme].name == phonemeName) { found = true; break; } } if (found) { double start = items[1].intervals[i].xmin / inputClip.length; double end = items[1].intervals[i].xmax / inputClip.length; double length = end - start; if ((length * inputClip.length) < minLengthForSustain) { data.Add(new PhonemeMarker(phoneme, (float)(start + (length / 2)))); } else { data.Add(new PhonemeMarker(phoneme, (float)start, 1, true)); data.Add(new PhonemeMarker(phoneme, (float)end)); } } else { Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry."); } } else { Debug.LogWarning("Phoneme mapper does not contain '" + label + "' Skipping this entry."); } } } } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Data loaded from MFA TextGrid file was invalid or incomplete.", ClipFeatures.None)); return; } inputClip.phonemeData = data.ToArray(); if (oovs.Count > 0) { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "Completed, but some words were not found. Check the console.", GetOutputCompatibility())); } else { callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "", GetOutputCompatibility())); } return; }
public override void ApplySetupWizardValues() { EditorPrefs.SetString("as_montrealfa_application_path", applicationPath); EditorPrefs.SetBool("as_montrealfa_application_path_verified", AutoSyncUtility.VerifyProgramAtPath(applicationPath, "align")); }
private bool Verify() { return(AutoSyncUtility.VerifyProgramAtPath(EditorPrefs.GetString("as_montrealfa_application_path"), "align")); }