public static string[] FindModels(string filter)
{
string[] assets = AssetDatabase.FindAssets("t:ASPocketSphinxLanguageModel " + filter);
for (int s = 0; s < assets.Length; s++)
{
ASPocketSphinxLanguageModel model = AssetDatabase.LoadAssetAtPath <ASPocketSphinxLanguageModel>(AssetDatabase.GUIDToAssetPath(assets[s]));
assets[s] = model.language;
}
return(assets);
}
public static ASPocketSphinxLanguageModel Load(int index)
{
string[] languageModelGUIDs = AssetDatabase.FindAssets("t:ASPocketSphinxLanguageModel");
var settings = LipSyncEditorExtensions.GetProjectFile();
if (settings == null)
{
return(null);
}
if (settings.phonemeSet == null)
{
return(null);
}
if (languageModelGUIDs.Length > 0 && languageModelGUIDs.Length > index)
{
ASPocketSphinxLanguageModel model = AssetDatabase.LoadAssetAtPath <ASPocketSphinxLanguageModel>(AssetDatabase.GUIDToAssetPath(languageModelGUIDs[index]));
if (model != null)
{
if (model.mappingMode == AutoSyncPhonemeMap.MappingMode.InternalMap && !string.IsNullOrEmpty(model.recommendedPhonemeSet) && model.recommendedPhonemeSet != settings.phonemeSet.scriptingName)
{
if (!EditorUtility.DisplayDialog("Wrong Phoneme Set", "Warning: You are using the '" + settings.phonemeSet.scriptingName + "' Phoneme Set, and this language model is designed for use with '" + model.recommendedPhonemeSet + "'. This may not provide usable results, are you sure you want to continue?", "Yes", "No"))
{
return(null);
}
}
return(model);
}
}
else
{
Debug.LogError("LipSync: Invalid PocketSphinx language model index provided.");
}
return(null);
}
public override void Process(LipSyncData inputClip, AutoSync.ASProcessDelegate callback)
{
bool converted = false;
string audioPath = AssetDatabase.GetAssetPath(inputClip.clip).Substring("/Assets".Length);
if (audioPath != null)
{
// Get absolute path
audioPath = Application.dataPath + "/" + audioPath;
// Check Path
if (audioPath.IndexOfAny(Path.GetInvalidPathChars()) >= 0 || Path.GetFileNameWithoutExtension(audioPath).IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio path contains invalid characters.", ClipFeatures.None));
return;
}
bool failed = false;
if (AutoSyncConversionUtility.IsConversionAvailable && useAudioConversion)
{
converted = true;
string newPath = Path.ChangeExtension(audioPath, ".converted.wav");
if (!AutoSyncConversionUtility.StartConversion(audioPath, newPath, AutoSyncConversionUtility.AudioFormat.WavPCM, 16000, 16, 1))
{
failed = true;
}
audioPath = newPath;
}
if (!File.Exists(audioPath) || failed)
{
if (converted)
{
if (File.Exists(audioPath))
{
File.Delete(audioPath);
AssetDatabase.Refresh();
}
}
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Audio conversion failed or file was deleted.", ClipFeatures.None));
return;
}
// Load Language Model
ASPocketSphinxLanguageModel model = ASPocketSphinxLanguageModel.Load(languageModel);
if (model == null)
{
if (converted)
{
if (File.Exists(audioPath))
{
File.Delete(audioPath);
AssetDatabase.Refresh();
}
}
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, "Language Model failed to load.", ClipFeatures.None));
return;
}
string basePath = model.GetBasePath();
List <string> args = new List <string>();
args.Add("-infile");
args.Add(audioPath);
args.Add("-hmm");
args.Add(basePath + model.hmmDir);
args.Add("-allphone");
args.Add(basePath + model.allphoneFile);
if (allphone_ciEnabled)
{
args.Add("-allphone_ci"); args.Add("yes");
}
if (backtraceEnabled)
{
args.Add("-backtrace"); args.Add("yes");
}
args.Add("-time");
args.Add("yes");
args.Add("-beam");
args.Add("1e" + beamExponent);
args.Add("-pbeam");
args.Add("1e" + pbeamExponent);
args.Add("-lw");
args.Add(lwValue.ToString());
SphinxWrapper.Recognize(args.ToArray());
ContinuationManager.Add(() => SphinxWrapper.isFinished, () =>
{
if (SphinxWrapper.error != null)
{
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(false, SphinxWrapper.error, ClipFeatures.None));
return;
}
List <PhonemeMarker> data = ParseOutput(
SphinxWrapper.result.Split(new string[] { "\r\n", "\n" }, StringSplitOptions.RemoveEmptyEntries),
model,
inputClip.clip
);
inputClip.phonemeData = data.ToArray();
callback.Invoke(inputClip, new AutoSync.ASProcessDelegateData(true, "", GetOutputCompatibility()));
if (converted)
{
if (File.Exists(audioPath))
{
File.Delete(audioPath);
AssetDatabase.Refresh();
}
}
});
}
}
private List <PhonemeMarker> ParseOutput(string[] lines, ASPocketSphinxLanguageModel lm, AudioClip clip)
{
List <PhonemeMarker> results = new List <PhonemeMarker>();
Dictionary <string, string> phonemeMapper = null;
var settings = LipSyncEditorExtensions.GetProjectFile();
bool needsMapping = true;
if (lm.sourcePhoneticAlphabetName == settings.phonemeSet.scriptingName)
{
needsMapping = false;
}
else
{
needsMapping = true;
switch (lm.mappingMode)
{
case AutoSyncPhonemeMap.MappingMode.InternalMap:
phonemeMapper = lm.phonemeMap.GenerateAtoBDictionary();
break;
case AutoSyncPhonemeMap.MappingMode.ExternalMap:
if (lm.externalMap)
{
phonemeMapper = lm.externalMap.phonemeMap.GenerateAtoBDictionary();
}
else
{
Debug.LogError("Language Model specifies an external phoneme map, but no phoneme map was provided.");
return(null);
}
break;
default:
case AutoSyncPhonemeMap.MappingMode.AutoDetect:
phonemeMapper = AutoSyncUtility.FindBestFitPhonemeMap(lm.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName);
if (phonemeMapper == null)
{
Debug.LogErrorFormat("No PhonemeMap could be found to map from '{0}' to the current PhonemeSet '{1}'.", lm.sourcePhoneticAlphabetName, settings.phonemeSet.scriptingName);
return(null);
}
break;
}
if (phonemeMapper.Count == 0)
{
Debug.LogWarning("PhonemeMap is empty - this may be due to the language model's mapping mode being set to 'InternalMap' but with no entries being added to the map. Phonemes may not be generated.");
}
}
NumberStyles style = NumberStyles.Number;
CultureInfo culture = CultureInfo.InvariantCulture;
foreach (string line in lines)
{
if (string.IsNullOrEmpty(line))
{
break;
}
string[] tokens = line.Split(' ');
try
{
if (tokens[0] != "SIL")
{
string phonemeName = needsMapping ? phonemeMapper[tokens[0]] : tokens[0];
float startTime = -1;
float.TryParse(tokens[1], style, culture, out startTime);
if (startTime > -1)
{
startTime /= clip.length;
}
else
{
Debug.LogWarning("Phoneme mapper returned invalid timestamp. Skipping this entry.");
continue;
}
bool found = false;
int phoneme;
for (phoneme = 0; phoneme < settings.phonemeSet.phonemes.Length; phoneme++)
{
if (settings.phonemeSet.phonemes[phoneme].name == phonemeName)
{
found = true;
break;
}
}
if (found)
{
results.Add(new PhonemeMarker(phoneme, startTime));
}
else
{
Debug.LogWarning("Phoneme mapper returned '" + phonemeName + "' but this phoneme does not exist in the current set. Skipping this entry.");
}
}
}
catch (ArgumentOutOfRangeException)
{
Debug.LogWarning("Phoneme Label missing from return data. Skipping this entry.");
}
catch (KeyNotFoundException)
{
Debug.LogWarning("Phoneme Label '" + tokens[0] + "' not found in phoneme mapper. Skipping this entry.");
}
}
return(results);
}
private void OnEnable()
{
languageModelNames = ASPocketSphinxLanguageModel.FindModels();
}