public override void Process(LipSyncData inputClip, AutoSync.ASProcessDelegate callback)
        {
            bool   converted = false;
            string audioPath = AssetDatabase.GetAssetPath(inputClip.clip).Substring("/Assets".Length);

            if (audioPath != null)
            {
                // Get absolute path
                audioPath = Application.dataPath + "/" + audioPath;

                // Check Path
                if (audioPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0 || Path.GetFileNameWithoutExtension(audioPath).IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
                {
                    callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path contains invalid characters.", ClipFeatures.None));
                    return;
                }

                bool failed = false;

                if (AutoSyncConversionUtility.IsConversionAvailable && useAudioConversion)
                {
                    converted = true;
                    string newPath = Path.ChangeExtension(audioPath, ".converted.wav");
                    if (!AutoSyncConversionUtility.StartConversion(audioPath, newPath, AutoSyncConversionUtility.AudioFormat.WavPCM, 16000, 16, 1))
                    {
                        failed = true;
                    }
                    audioPath = newPath;
                }

                if (!File.Exists(audioPath) || failed)
                {
                    if (converted)
                    {
                        if (File.Exists(audioPath))
                        {
                            File.Delete(audioPath);
                            AssetDatabase.Refresh();
                        }
                    }

                    callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio conversion failed or file was deleted.", ClipFeatures.None));
                    return;
                }

                // Load Language Model
                ASPocketSphinxLanguageModel model = ASPocketSphinxLanguageModel.Load(languageModel);
                if (model == null)
                {
                    if (converted)
                    {
                        if (File.Exists(audioPath))
                        {
                            File.Delete(audioPath);
                            AssetDatabase.Refresh();
                        }
                    }
                    callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model failed to load.", ClipFeatures.None));
                    return;
                }
                string basePath = model.GetBasePath();

                List <string> args = new List <string>();
                args.Add("-infile");
                args.Add(audioPath);
                args.Add("-hmm");
                args.Add(basePath + model.hmmDir);
                args.Add("-allphone");
                args.Add(basePath + model.allphoneFile);
                if (allphone_ciEnabled)
                {
                    args.Add("-allphone_ci"); args.Add("yes");
                }
                if (backtraceEnabled)
                {
                    args.Add("-backtrace"); args.Add("yes");
                }
                args.Add("-time");
                args.Add("yes");
                args.Add("-beam");
                args.Add("1e" + beamExponent);
                args.Add("-pbeam");
                args.Add("1e" + pbeamExponent);
                args.Add("-lw");
                args.Add(lwValue.ToString());

                SphinxWrapper.Recognize(args.ToArray());

                ContinuationManager.Add(() => SphinxWrapper.isFinished, () =>
                {
                    if (SphinxWrapper.error != null)
                    {
                        callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, SphinxWrapper.error, ClipFeatures.None));
                        return;
                    }

                    List <PhonemeMarker> data = ParseOutput(
                        SphinxWrapper.result.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries),
                        model,
                        inputClip.clip
                        );

                    inputClip.phonemeData = data.ToArray();
                    callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "", GetOutputCompatibility()));

                    if (converted)
                    {
                        if (File.Exists(audioPath))
                        {
                            File.Delete(audioPath);
                            AssetDatabase.Refresh();
                        }
                    }
                });
            }
        }
Exemple #2
0
        public override void Process(LipSyncData inputClip, AutoSync.ASProcessDelegate callback)
        {
            string mfaPath   = EditorPrefs.GetString("as_montrealfa_application_path");
            string audioPath = AssetDatabase.GetAssetPath(inputClip.clip).Substring("/Assets".Length);

            if (audioPath == null)
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path could not be found.", ClipFeatures.None));
                return;
            }

            if (!AutoSyncUtility.VerifyProgramAtPath(mfaPath, "align"))
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Montreal Forced Aligner application path is not verified.", ClipFeatures.None));
                return;
            }

            // Get absolute path
            audioPath = Application.dataPath + "/" + audioPath;

            // Check Path
            if (audioPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0 || Path.GetFileNameWithoutExtension(audioPath).IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path contains invalid characters.", ClipFeatures.None));
                return;
            }

            // Load Language Model
            ASMontrealLanguageModel model = ASMontrealLanguageModel.Load(languageModel);

            if (model == null)
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model failed to load.", ClipFeatures.None));
                return;
            }

            string basePath    = model.GetBasePath();
            string lexiconPath = "";

            if (model.usePredefinedLexicon)
            {
                lexiconPath = basePath + model.lexiconPath;
            }
            else
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Support for generated lexicons using a G2P model is coming soon.", ClipFeatures.None));
                return;
            }

            string adjustedName = Path.GetFileNameWithoutExtension(audioPath).Replace('.', '_').Replace(' ', '_').Replace('\\', '_').Replace('/', '_');

            string corpusPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Corpus";
            string outputPath = Application.temporaryCachePath + "/" + adjustedName + "_MFA_Output";

            // Delete folders if they already exist
            try
            {
                if (Directory.Exists(corpusPath))
                {
                    Directory.Delete(corpusPath, true);
                }
                if (Directory.Exists(outputPath))
                {
                    Directory.Delete(outputPath, true);
                }
            }
            catch
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Attempt to clear temporary MFA folders failed. Are they open in another application?", ClipFeatures.None));
                return;
            }

            // Create temporary folders
            Directory.CreateDirectory(corpusPath);
            Directory.CreateDirectory(outputPath);

            // Copy or convert audio clip to corpus folder
            if (AutoSyncConversionUtility.IsConversionAvailable && useAudioConversion)
            {
                AutoSyncConversionUtility.StartConversion(audioPath, corpusPath + "/" + adjustedName + ".wav", AutoSyncConversionUtility.AudioFormat.WavPCM, 16000, 16, 1);
            }
            else
            {
                File.Copy(audioPath, corpusPath + "/" + adjustedName + Path.GetExtension(audioPath));
            }

            // Create transcript file in corpus folder
            StreamWriter transcriptWriter = File.CreateText(corpusPath + "/" + adjustedName + ".lab");

            transcriptWriter.Write(inputClip.transcript.Replace('-', ' '));
            transcriptWriter.Close();

            // Run aligner
            Directory.SetCurrentDirectory(Application.dataPath.Remove(Application.dataPath.Length - 6));
            mfaPath = Path.GetFullPath(mfaPath);

            System.Diagnostics.Process process = new System.Diagnostics.Process();
            process.StartInfo.FileName = mfaPath;

#if UNITY_EDITOR_WIN
            process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\" --quiet";
#elif UNITY_EDITOR_OSX
            process.StartInfo.Arguments = "\"" + corpusPath + "\" \"" + lexiconPath + "\" \"" + basePath + model.acousticModelPath + "\" \"" + outputPath + "\"";
#endif
            process.StartInfo.UseShellExecute = true;
            process.StartInfo.CreateNoWindow  = true;

            process.Start();
            process.WaitForExit(15000);

            if (!process.HasExited)
            {
                process.Kill();
                process.Close();
            }

            var           outputFiles  = Directory.GetFiles(outputPath, "*", SearchOption.AllDirectories);
            string        textGridPath = "";
            List <string> oovs         = new List <string>();
            bool          nothingFound = true;

            for (int i = 0; i < outputFiles.Length; i++)
            {
                if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".textgrid")
                {
                    textGridPath = outputFiles[i];
                    nothingFound = false;
                }
                else if (Path.GetExtension(outputFiles[i]).ToLowerInvariant() == ".txt")
                {
                    string name   = Path.GetFileNameWithoutExtension(outputFiles[i]);
                    var    reader = new StreamReader(outputFiles[i]);
                    if (name == "oovs_found")
                    {
                        while (!reader.EndOfStream)
                        {
                            oovs.Add(reader.ReadLine());
                        }
                    }
                    reader.Close();
                }
            }

            // Detect out-of-vocab words, filter and retry if enabled.
            if (oovs.Count > 0)
            {
                Debug.Log("Found out-of-vocabulary words:");
                for (int i = 0; i < oovs.Count; i++)
                {
                    Debug.Log(oovs[i]);
                }
            }

            if (nothingFound)
            {
                if (autoRetry)
                {
                    if (attempts < maxAttempts - 1)
                    {
                        attempts++;
                        Process(inputClip, callback);
                        return;
                    }
                }

                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "MFA Application Failed. Check your audio encoding or enable conversion.", ClipFeatures.None));
                return;
            }

            // Load in TextGrid
            TextGridUtility.TextGridItem[] items = TextGridUtility.ParseTextGridFile(textGridPath);

            var  settings     = LipSyncEditorExtensions.GetProjectFile();
            bool needsMapping = true;
            Dictionary <string, string> phonemeMapper = null;

            if (model.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName)
            {
                needsMapping = false;
            }
            else
            {
                needsMapping = true;
                switch (model.mappingMode)
                {
                case AutoSyncPhonemeMap.MappingMode.InternalMap:
                    phonemeMapper = model.phonemeMap.GenerateAtoBDictionary();
                    break;

                case AutoSyncPhonemeMap.MappingMode.ExternalMap:
                    if (model.externalMap)
                    {
                        phonemeMapper = model.externalMap.phonemeMap.GenerateAtoBDictionary();
                    }
                    else
                    {
                        callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model specifies an external phoneme map, but no phoneme map was provided.", ClipFeatures.None));
                        return;
                    }
                    break;

                default:
                case AutoSyncPhonemeMap.MappingMode.AutoDetect:
                    phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName);
                    if (phonemeMapper == null)
                    {
                        callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, string.Format("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", model.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName), ClipFeatures.None));
                        return;
                    }
                    break;
                }

                if (phonemeMapper.Count == 0)
                {
                    Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated.");
                }
            }

            // Get Phones
            List <PhonemeMarker> data = new List <PhonemeMarker>();

            if (items != null && items.Length == 2)
            {
                for (int i = 0; i < items[1].intervals.Length; i++)
                {
                    if (items[1].intervals[i] == null)
                    {
                        Debug.LogFormat("Interval {0} is null :o", i);
                        continue;
                    }

                    string label = items[1].intervals[i].text.Split('"')[1];
                    label = System.Text.RegularExpressions.Regex.Replace(label, "[0-9]", "");

                    if (label != "sil")
                    {
                        if (phonemeMapper.ContainsKey(label))
                        {
                            string phonemeName = needsMapping ? phonemeMapper[label] : label;

                            bool found = false;
                            int  phoneme;
                            for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++)
                            {
                                if (settings.phonemeSet.phonemes[phoneme].name == phonemeName)
                                {
                                    found = true;
                                    break;
                                }
                            }

                            if (found)
                            {
                                double start = items[1].intervals[i].xmin / inputClip.length;
                                double end   = items[1].intervals[i].xmax / inputClip.length;

                                double length = end - start;
                                if ((length * inputClip.length) < minLengthForSustain)
                                {
                                    data.Add(new PhonemeMarker(phoneme, (float)(start + (length / 2))));
                                }
                                else
                                {
                                    data.Add(new PhonemeMarker(phoneme, (float)start, 1, true));
                                    data.Add(new PhonemeMarker(phoneme, (float)end));
                                }
                            }
                            else
                            {
                                Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry.");
                            }
                        }
                        else
                        {
                            Debug.LogWarning("Phoneme mapper does not contain '" + label + "' Skipping this entry.");
                        }
                    }
                }
            }
            else
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Data loaded from MFA TextGrid file was invalid or incomplete.", ClipFeatures.None));
                return;
            }


            inputClip.phonemeData = data.ToArray();

            if (oovs.Count > 0)
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "Completed, but some words were not found. Check the console.", GetOutputCompatibility()));
            }
            else
            {
                callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "", GetOutputCompatibility()));
            }

            return;
        }