void Update() { var position = Microphone.GetPosition(null); if (position < 0 || head == position) { return; } clip.GetData(microphoneBuffer, 0); while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length) { var remain = microphoneBuffer.Length - head; if (remain < processBuffer.Length) { Array.Copy(microphoneBuffer, head, processBuffer, 0, remain); Array.Copy(microphoneBuffer, 0, processBuffer, remain, processBuffer.Length - remain); } else { Array.Copy(microphoneBuffer, head, processBuffer, 0, processBuffer.Length); } OVRLipSync.ProcessFrame(Context, processBuffer, Frame); head += processBuffer.Length; if (head > microphoneBuffer.Length) { head -= microphoneBuffer.Length; } } }
/// <summary> /// Pass an audio sample to the lip sync module for computation /// </summary> /// <param name="data">Data.</param> /// <param name="channels">Channels.</param> public void ProcessAudioSamples(float[] data, int channels) { // Do not process if we are not initialized, or if there is no // audio source attached to game object if ((OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) || audioSource == null) { return; } // Increase the gain of the input for (int i = 0; i < data.Length; ++i) { data[i] = data[i] * gain; } // Send data into Phoneme context for processing (if context is not 0) lock (this) { if (Context != 0) { OVRLipSync.Frame frame = this.Frame; OVRLipSync.ProcessFrame(Context, data, frame); } } // Turn off output (so that we don't get feedback from mics too close to speakers) if (!audioLoopback) { for (int i = 0; i < data.Length; ++i) { data[i] = data[i] * 0.0f; } } }
private void Update() { if (!IsRecording) { return; } int position = Microphone.GetPosition(DeviceName); //読み取り位置がずっと動かない場合、マイクを復帰させる。PS4コンを挿抜したときにマイクが勝手に止まる事があります…。 if (position == _prevPosition) { _positionNotMovedCount++; if (_positionNotMovedCount > PositionStopCountLimit) { _positionNotMovedCount = 0; RestartMicrophone(); return; } } else { _prevPosition = position; _positionNotMovedCount = 0; } //マイクの動いてる/動かないの検知とは別で範囲チェック if (position < 0 || _head == position) { return; } _clip.GetData(_microphoneBuffer, 0); while (GetDataLength(_microphoneBuffer.Length, _head, position) > _processBuffer.Length) { var remain = _microphoneBuffer.Length - _head; if (remain < _processBuffer.Length) { Array.Copy(_microphoneBuffer, _head, _processBuffer, 0, remain); Array.Copy(_microphoneBuffer, 0, _processBuffer, remain, _processBuffer.Length - remain); } else { Array.Copy(_microphoneBuffer, _head, _processBuffer, 0, _processBuffer.Length); } OVRLipSync.ProcessFrame(Context, _processBuffer, Frame); _head += _processBuffer.Length; if (_head > _microphoneBuffer.Length) { _head -= _microphoneBuffer.Length; } } }
/// <summary> /// Pass S16 PCM audio buffer to the lip sync module /// </summary> /// <param name="data">Data.</param> /// <param name="channels">Channels.</param> public void ProcessAudioSamplesRaw(short[] data, int channels) { // Send data into Phoneme context for processing (if context is not 0) lock (this) { if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) { return; } var frame = this.Frame; OVRLipSync.ProcessFrame(Context, data, frame, channels == 2); } }
void Update() { if (clip == null) { return; } var position = Microphone.GetPosition(selectedDevice); if (position < 0 || head == position) { return; } clip.GetData(microphoneBuffer, 0); while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length) { var remain = microphoneBuffer.Length - head; if (remain < processBuffer.Length) { for (int i = 0; i < remain; i++) { processBuffer[i] = microphoneBuffer[head + i] * gain; } for (int i = remain; i < processBuffer.Length - remain; i++) { processBuffer[i] = microphoneBuffer[i - remain] * gain; } } else { for (int i = 0; i < processBuffer.Length; i++) { processBuffer[i] = microphoneBuffer[head + i] * gain; } } OVRLipSync.ProcessFrame(Context, processBuffer, Frame, false); head += processBuffer.Length; if (head > microphoneBuffer.Length) { head -= microphoneBuffer.Length; } } }
/// <summary> /// Pass F32 PCM audio buffer to the lip sync module /// </summary> /// <param name="data">Data.</param> /// <param name="channels">Channels.</param> public void ProcessAudioSamplesRaw(float[] data, int channels) { // Send data into Phoneme context for processing (if context is not 0) lock (this) { if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) { Debug.LogError("context is " + Context); Debug.LogError("OVRLipSync.IsInitialized() " + OVRLipSync.IsInitialized().ToString()); return; } var frame = this.Frame; OVRLipSync.Result result = OVRLipSync.ProcessFrame(Context, data, frame, channels == 2); //Debug.Log("result is " + frame.frameNumber); } }
void OnAudioFilterRead(float[] data, int channels) { if (EnableLowLatency == false) { // Do not spatialize if we are not initialized, or if there is no // audio source attached to game object if ((OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) || audioSource == null) { return; } // increase the gain of the input to get a better signal input for (int i = 0; i < data.Length; ++i) { data[i] = data[i] * Gain; } // Send data into Phoneme context for processing (if context is not 0) lock (this) { if (Context != 0) { //OVRLipSync.Flags flags = 0; // Set flags to feed into process //if (DelayCompensate == true) // flags |= OVRLipSync.Flags.DelayCompensateAudio; OVRLipSync.Frame frame = this.Frame; OVRLipSync.ProcessFrame(Context, data, /* flags,*/ frame); } } } // Turn off output (so that we don't get feedback from mics too close to speakers) if (AudioMute == true) { for (int i = 0; i < data.Length; ++i) { data[i] = data[i] * 0.0f; } } }
/// <summary> /// マイク入力専用音データ取得 /// OnAudioFilterRead を使うより低遅延 /// </summary> void ProcessMicrophoneAudioReadFast() { if (!micSelected) { return; } if (!Microphone.IsRecording(selectedDevice)) { StartMicrophone(); } var position = Microphone.GetPosition(selectedDevice); if (position < 0 || head == position) { return; } audioSource.clip.GetData(microphoneBuffer, 0); while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length) { var remain = microphoneBuffer.Length - head; if (remain < processBuffer.Length) { Array.Copy(microphoneBuffer, head, processBuffer, 0, remain); Array.Copy(microphoneBuffer, 0, processBuffer, remain, processBuffer.Length - remain); } else { Array.Copy(microphoneBuffer, head, processBuffer, 0, processBuffer.Length); } OVRLipSync.ProcessFrame(Context, processBuffer, Frame); head += processBuffer.Length; if (head > microphoneBuffer.Length) { head -= microphoneBuffer.Length; } //maxVolume = Math.Max (processBuffer.Max (), maxVolume); } }
public static OVRLipSyncSequence CreateSequenceFromAudioClip( AudioClip clip, bool useOfflineModel = false) { OVRLipSyncSequence sequence = null; if (clip.channels > 2) { Debug.LogError(clip.name + ": Cannot process phonemes from an audio clip with " + "more than 2 channels"); return(null); } if (clip.loadType != AudioClipLoadType.DecompressOnLoad) { Debug.LogError(clip.name + ": Cannot process phonemes from an audio clip unless " + "its load type is set to DecompressOnLoad."); return(null); } if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success) { Debug.LogError("Could not create Lip Sync engine."); return(null); } if (clip.loadState != AudioDataLoadState.Loaded) { Debug.LogError("Clip is not loaded!"); return(null); } uint context = 0; OVRLipSync.Result result = useOfflineModel ? OVRLipSync.CreateContextWithModelFile( ref context, OVRLipSync.ContextProviders.Enhanced, Path.Combine(Application.dataPath, "Oculus/LipSync/Assets/OfflineModel/ovrlipsync_offline_model.pb")) : OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Enhanced); if (result != OVRLipSync.Result.Success) { Debug.LogError("Could not create Phoneme context. (" + result + ")"); OVRLipSync.Shutdown(); return(null); } List <OVRLipSync.Frame> frames = new List <OVRLipSync.Frame>(); float[] samples = new float[sSampleSize * clip.channels]; OVRLipSync.Frame dummyFrame = new OVRLipSync.Frame(); OVRLipSync.ProcessFrame( context, samples, dummyFrame, (clip.channels == 2) ? true : false ); // frame delay in ms float frameDelayInMs = dummyFrame.frameDelay; int frameOffset = (int)(frameDelayInMs * clip.frequency / 1000); int totalSamples = clip.samples; for (int x = 0; x < totalSamples + frameOffset; x += sSampleSize) { int remainingSamples = totalSamples - x; if (remainingSamples >= sSampleSize) { clip.GetData(samples, x); } else if (remainingSamples > 0) { float[] samples_clip = new float[remainingSamples * clip.channels]; clip.GetData(samples_clip, x); Array.Copy(samples_clip, samples, samples_clip.Length); Array.Clear(samples, samples_clip.Length, samples.Length - samples_clip.Length); } else { Array.Clear(samples, 0, samples.Length); } OVRLipSync.Frame frame = new OVRLipSync.Frame(); if (clip.channels == 2) { // interleaved = stereo data, alternating floats OVRLipSync.ProcessFrame(context, samples, frame); } else { // mono OVRLipSync.ProcessFrame(context, samples, frame, false); } if (x < frameOffset) { continue; } frames.Add(frame); } Debug.Log(clip.name + " produced " + frames.Count + " viseme frames, playback rate is " + (frames.Count / clip.length) + " fps"); OVRLipSync.DestroyContext(context); OVRLipSync.Shutdown(); sequence = ScriptableObject.CreateInstance <OVRLipSyncSequence>(); sequence.entries = frames; sequence.length = clip.length; return(sequence); }
// Update is called once per frame void Update() { if (EnableLipSync) { if (Context != 0) { if (proxy == null) { if (VRMmodel != null) { proxy = VRMmodel.GetComponent <VRMBlendShapeProxy>(); } } else { // get the current viseme frame OVRLipSync.Frame frame = GetCurrentPhonemeFrame(); if (frame != null) { //あ OVRLipSync.Viseme.aa; BlendShapePreset.A; //い OVRLipSync.Viseme.ih; BlendShapePreset.I; //う OVRLipSync.Viseme.ou; BlendShapePreset.U; //え OVRLipSync.Viseme.E; BlendShapePreset.E; //お OVRLipSync.Viseme.oh; BlendShapePreset.O; var presets = new BlendShapePreset[] { BlendShapePreset.A, BlendShapePreset.I, BlendShapePreset.U, BlendShapePreset.E, BlendShapePreset.O, }; var visemes = new float[] { frame.Visemes[(int)OVRLipSync.Viseme.aa], frame.Visemes[(int)OVRLipSync.Viseme.ih], frame.Visemes[(int)OVRLipSync.Viseme.ou], frame.Visemes[(int)OVRLipSync.Viseme.E], frame.Visemes[(int)OVRLipSync.Viseme.oh], }; int maxindex = 0; float maxvisemes = 0; for (int i = 0; i < presets.Length; i++) { if (visemes[i] < WeightThreashold) { visemes[i] = 0; } if (maxvisemes < visemes[i]) { maxindex = i; maxvisemes = visemes[i]; } } if (MaxWeightEmphasis) { visemes[maxindex] = Mathf.Clamp(visemes[maxindex] * 3, 0.0f, 1.0f); } if (MaxWeightEnable) { for (int i = 0; i < presets.Length; i++) { if (i != maxindex) { visemes[i] = 0.0f; } } } for (int i = 0; i < presets.Length; i++) { visemes[i] *= MaxLevel; proxy.SetValue(presets[i], visemes[i]); } //Debug.Log("Visemes:" + string.Join(",", frame.Visemes.Select(d => d.ToString()))); } } } if (string.IsNullOrEmpty(selectedDevice) == false) { audioSource.volume = (sourceVolume / 100); if (!Microphone.IsRecording(selectedDevice)) { StartMicrophone(); } if (EnableLowLatency) { var position = Microphone.GetPosition(selectedDevice); if (position < 0 || head == position) { return; } audioSource.clip.GetData(microphoneBuffer, 0); while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length) { var remain = microphoneBuffer.Length - head; if (remain < processBuffer.Length) { Array.Copy(microphoneBuffer, head, processBuffer, 0, remain); Array.Copy(microphoneBuffer, 0, processBuffer, remain, processBuffer.Length - remain); } else { Array.Copy(microphoneBuffer, head, processBuffer, 0, processBuffer.Length); } OVRLipSync.ProcessFrame(Context, processBuffer, Frame); head += processBuffer.Length; if (head > microphoneBuffer.Length) { head -= microphoneBuffer.Length; } } } } } }
public static OVRLipSyncSequence CreateSequenceFromAudioClip(AudioClip clip) { OVRLipSyncSequence sequence = null; if (clip.loadType != AudioClipLoadType.DecompressOnLoad || clip.channels > 2) { // todo: just fix the clip Debug.LogError("Cannot process phonemes from an audio clip unless its load type is set to DecompressOnLoad."); } else { if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success) { Debug.LogError("Could not create Lip Sync engine."); } else { uint context = 0; OVRLipSync.Result result = OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Main); if (result != OVRLipSync.Result.Success) { Debug.LogError("Could not create Phoneme context. (" + result + ")"); OVRLipSync.Shutdown(); } else { List <OVRLipSync.Frame> frames = new List <OVRLipSync.Frame>(); float[] samples = new float[sSampleSize * clip.channels]; int totalSamples = clip.samples; for (int x = 0; x < totalSamples; x += sSampleSize) { // GetData loops at the end of the read. Prevent that when it happens. if (x + samples.Length > totalSamples) { samples = new float[(totalSamples - x) * clip.channels]; } clip.GetData(samples, x); OVRLipSync.Frame frame = new OVRLipSync.Frame(); if (clip.channels == 2) { // interleaved = stereo data, alternating floats OVRLipSync.ProcessFrameInterleaved(context, samples, 0, frame); } else { // mono OVRLipSync.ProcessFrame(context, samples, 0, frame); } frames.Add(frame); } Debug.Log(clip.name + " produced " + frames.Count + " viseme frames, playback rate is " + (frames.Count / clip.length) + " fps"); OVRLipSync.DestroyContext(context); OVRLipSync.Shutdown(); sequence = ScriptableObject.CreateInstance <OVRLipSyncSequence>(); sequence.entries = frames; sequence.length = clip.length; } } } return(sequence); }
void ProcessBuffer(float[] buffer) { if (buffer == null) { return; } audioVolume = 0f; foreach (float v in buffer) { audioVolume += Mathf.Abs(v); } audioVolume /= buffer.Length; int totalLen = partialPos + buffer.Length; int bufferPos = 0; if (totalLen >= 1024 * channels) { volume = 0f; while (totalLen >= 1024 * channels) { int sliceLen = 1024 - partialPos; Array.Copy(buffer, bufferPos, partialAudio, partialPos, sliceLen * channels); totalLen -= 1024 * channels; if (totalLen < 1024 * channels) { for (int i = 0; i < partialAudio.Length; i++) { partialAudio[i] = partialAudio[i] * gain;//Mathf.Clamp(partialAudio[i] * gain, 0f, 1f); volume += Mathf.Abs(partialAudio[i]); } lock (this) { if (context != 0) { OVRLipSync.Frame frame = this.visemeData; if (channels == 2) { OVRLipSync.ProcessFrameInterleaved(context, partialAudio, frame); } else { OVRLipSync.ProcessFrame(context, partialAudio, frame); } } else { Debug.Log("OVRLipSync context is 0"); } } } bufferPos += sliceLen; partialPos = 0; } volume /= (float)buffer.Length; } if (totalLen > 0) { Array.Copy(buffer, bufferPos, partialAudio, partialPos, buffer.Length - bufferPos); partialPos += (buffer.Length - bufferPos) / channels; } }
public static OVRLipSyncSequence CreateSequenceFromAudioClip( AudioClip clip, bool useOfflineModel = false) { OVRLipSyncSequence sequence = null; if (clip.channels > 2) { Debug.LogError(clip.name + ": Cannot process phonemes from an audio clip with " + "more than 2 channels"); return(null); } if (clip.loadType != AudioClipLoadType.DecompressOnLoad) { Debug.LogError(clip.name + ": Cannot process phonemes from an audio clip unless " + "its load type is set to DecompressOnLoad."); return(null); } if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success) { Debug.LogError("Could not create Lip Sync engine."); return(null); } if (clip.loadState != AudioDataLoadState.Loaded) { Debug.LogError("Clip is not loaded!"); return(null); } uint context = 0; OVRLipSync.Result result = useOfflineModel ? OVRLipSync.CreateContextWithModelFile( ref context, OVRLipSync.ContextProviders.Enhanced, Path.Combine(Application.dataPath, "Oculus/LipSync/Assets/OfflineModel/ovrlipsync_offline_model.pb")) : OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Enhanced); if (result != OVRLipSync.Result.Success) { Debug.LogError("Could not create Phoneme context. (" + result + ")"); OVRLipSync.Shutdown(); return(null); } List <OVRLipSync.Frame> frames = new List <OVRLipSync.Frame>(); float[] samples = new float[sSampleSize * clip.channels]; int totalSamples = clip.samples; for (int x = 0; x < totalSamples; x += sSampleSize) { // GetData loops at the end of the read. Prevent that when it happens. if (x + samples.Length > totalSamples) { samples = new float[(totalSamples - x) * clip.channels]; } clip.GetData(samples, x); OVRLipSync.Frame frame = new OVRLipSync.Frame(); if (clip.channels == 2) { // interleaved = stereo data, alternating floats OVRLipSync.ProcessFrame(context, samples, frame); } else { // mono OVRLipSync.ProcessFrame(context, samples, frame, false); } frames.Add(frame); } Debug.Log(clip.name + " produced " + frames.Count + " viseme frames, playback rate is " + (frames.Count / clip.length) + " fps"); OVRLipSync.DestroyContext(context); OVRLipSync.Shutdown(); sequence = ScriptableObject.CreateInstance <OVRLipSyncSequence>(); sequence.entries = frames; sequence.length = clip.length; return(sequence); }