void Update()
{
var position = Microphone.GetPosition(null);
if (position < 0 || head == position)
{
return;
}
clip.GetData(microphoneBuffer, 0);
while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length)
{
var remain = microphoneBuffer.Length - head;
if (remain < processBuffer.Length)
{
Array.Copy(microphoneBuffer, head, processBuffer, 0, remain);
Array.Copy(microphoneBuffer, 0, processBuffer, remain, processBuffer.Length - remain);
}
else
{
Array.Copy(microphoneBuffer, head, processBuffer, 0, processBuffer.Length);
}
OVRLipSync.ProcessFrame(Context, processBuffer, Frame);
head += processBuffer.Length;
if (head > microphoneBuffer.Length)
{
head -= microphoneBuffer.Length;
}
}
}
/// <summary>
/// Pass an audio sample to the lip sync module for computation
/// </summary>
/// <param name="data">Data.</param>
/// <param name="channels">Channels.</param>
public void ProcessAudioSamples(float[] data, int channels)
{
// Do not process if we are not initialized, or if there is no
// audio source attached to game object
if ((OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) || audioSource == null)
{
return;
}
// Increase the gain of the input
for (int i = 0; i < data.Length; ++i)
{
data[i] = data[i] * gain;
}
// Send data into Phoneme context for processing (if context is not 0)
lock (this)
{
if (Context != 0)
{
OVRLipSync.Frame frame = this.Frame;
OVRLipSync.ProcessFrame(Context, data, frame);
}
}
// Turn off output (so that we don't get feedback from mics too close to speakers)
if (!audioLoopback)
{
for (int i = 0; i < data.Length; ++i)
{
data[i] = data[i] * 0.0f;
}
}
}
private void Update()
{
if (!IsRecording)
{
return;
}
int position = Microphone.GetPosition(DeviceName);
//読み取り位置がずっと動かない場合、マイクを復帰させる。PS4コンを挿抜したときにマイクが勝手に止まる事があります…。
if (position == _prevPosition)
{
_positionNotMovedCount++;
if (_positionNotMovedCount > PositionStopCountLimit)
{
_positionNotMovedCount = 0;
RestartMicrophone();
return;
}
}
else
{
_prevPosition = position;
_positionNotMovedCount = 0;
}
//マイクの動いてる/動かないの検知とは別で範囲チェック
if (position < 0 || _head == position)
{
return;
}
_clip.GetData(_microphoneBuffer, 0);
while (GetDataLength(_microphoneBuffer.Length, _head, position) > _processBuffer.Length)
{
var remain = _microphoneBuffer.Length - _head;
if (remain < _processBuffer.Length)
{
Array.Copy(_microphoneBuffer, _head, _processBuffer, 0, remain);
Array.Copy(_microphoneBuffer, 0, _processBuffer, remain, _processBuffer.Length - remain);
}
else
{
Array.Copy(_microphoneBuffer, _head, _processBuffer, 0, _processBuffer.Length);
}
OVRLipSync.ProcessFrame(Context, _processBuffer, Frame);
_head += _processBuffer.Length;
if (_head > _microphoneBuffer.Length)
{
_head -= _microphoneBuffer.Length;
}
}
}
/// <summary>
/// Pass S16 PCM audio buffer to the lip sync module
/// </summary>
/// <param name="data">Data.</param>
/// <param name="channels">Channels.</param>
public void ProcessAudioSamplesRaw(short[] data, int channels)
{
// Send data into Phoneme context for processing (if context is not 0)
lock (this)
{
if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success)
{
return;
}
var frame = this.Frame;
OVRLipSync.ProcessFrame(Context, data, frame, channels == 2);
}
}
void Update()
{
if (clip == null)
{
return;
}
var position = Microphone.GetPosition(selectedDevice);
if (position < 0 || head == position)
{
return;
}
clip.GetData(microphoneBuffer, 0);
while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length)
{
var remain = microphoneBuffer.Length - head;
if (remain < processBuffer.Length)
{
for (int i = 0; i < remain; i++)
{
processBuffer[i] = microphoneBuffer[head + i] * gain;
}
for (int i = remain; i < processBuffer.Length - remain; i++)
{
processBuffer[i] = microphoneBuffer[i - remain] * gain;
}
}
else
{
for (int i = 0; i < processBuffer.Length; i++)
{
processBuffer[i] = microphoneBuffer[head + i] * gain;
}
}
OVRLipSync.ProcessFrame(Context, processBuffer, Frame, false);
head += processBuffer.Length;
if (head > microphoneBuffer.Length)
{
head -= microphoneBuffer.Length;
}
}
}
/// <summary>
/// Pass F32 PCM audio buffer to the lip sync module
/// </summary>
/// <param name="data">Data.</param>
/// <param name="channels">Channels.</param>
public void ProcessAudioSamplesRaw(float[] data, int channels)
{
// Send data into Phoneme context for processing (if context is not 0)
lock (this)
{
if (Context == 0 || OVRLipSync.IsInitialized() != OVRLipSync.Result.Success)
{
Debug.LogError("context is " + Context);
Debug.LogError("OVRLipSync.IsInitialized() " + OVRLipSync.IsInitialized().ToString());
return;
}
var frame = this.Frame;
OVRLipSync.Result result = OVRLipSync.ProcessFrame(Context, data, frame, channels == 2);
//Debug.Log("result is " + frame.frameNumber);
}
}
void OnAudioFilterRead(float[] data, int channels)
{
if (EnableLowLatency == false)
{
// Do not spatialize if we are not initialized, or if there is no
// audio source attached to game object
if ((OVRLipSync.IsInitialized() != OVRLipSync.Result.Success) || audioSource == null)
{
return;
}
// increase the gain of the input to get a better signal input
for (int i = 0; i < data.Length; ++i)
{
data[i] = data[i] * Gain;
}
// Send data into Phoneme context for processing (if context is not 0)
lock (this)
{
if (Context != 0)
{
//OVRLipSync.Flags flags = 0;
// Set flags to feed into process
//if (DelayCompensate == true)
// flags |= OVRLipSync.Flags.DelayCompensateAudio;
OVRLipSync.Frame frame = this.Frame;
OVRLipSync.ProcessFrame(Context, data, /* flags,*/ frame);
}
}
}
// Turn off output (so that we don't get feedback from mics too close to speakers)
if (AudioMute == true)
{
for (int i = 0; i < data.Length; ++i)
{
data[i] = data[i] * 0.0f;
}
}
}
/// <summary>
/// マイク入力専用音データ取得
/// OnAudioFilterRead を使うより低遅延
/// </summary>
void ProcessMicrophoneAudioReadFast()
{
if (!micSelected)
{
return;
}
if (!Microphone.IsRecording(selectedDevice))
{
StartMicrophone();
}
var position = Microphone.GetPosition(selectedDevice);
if (position < 0 || head == position)
{
return;
}
audioSource.clip.GetData(microphoneBuffer, 0);
while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length)
{
var remain = microphoneBuffer.Length - head;
if (remain < processBuffer.Length)
{
Array.Copy(microphoneBuffer, head, processBuffer, 0, remain);
Array.Copy(microphoneBuffer, 0, processBuffer, remain, processBuffer.Length - remain);
}
else
{
Array.Copy(microphoneBuffer, head, processBuffer, 0, processBuffer.Length);
}
OVRLipSync.ProcessFrame(Context, processBuffer, Frame);
head += processBuffer.Length;
if (head > microphoneBuffer.Length)
{
head -= microphoneBuffer.Length;
}
//maxVolume = Math.Max (processBuffer.Max (), maxVolume);
}
}
public static OVRLipSyncSequence CreateSequenceFromAudioClip(
AudioClip clip, bool useOfflineModel = false)
{
OVRLipSyncSequence sequence = null;
if (clip.channels > 2)
{
Debug.LogError(clip.name +
": Cannot process phonemes from an audio clip with " +
"more than 2 channels");
return(null);
}
if (clip.loadType != AudioClipLoadType.DecompressOnLoad)
{
Debug.LogError(clip.name +
": Cannot process phonemes from an audio clip unless " +
"its load type is set to DecompressOnLoad.");
return(null);
}
if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success)
{
Debug.LogError("Could not create Lip Sync engine.");
return(null);
}
if (clip.loadState != AudioDataLoadState.Loaded)
{
Debug.LogError("Clip is not loaded!");
return(null);
}
uint context = 0;
OVRLipSync.Result result = useOfflineModel
? OVRLipSync.CreateContextWithModelFile(
ref context,
OVRLipSync.ContextProviders.Enhanced,
Path.Combine(Application.dataPath, "Oculus/LipSync/Assets/OfflineModel/ovrlipsync_offline_model.pb"))
: OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Enhanced);
if (result != OVRLipSync.Result.Success)
{
Debug.LogError("Could not create Phoneme context. (" + result + ")");
OVRLipSync.Shutdown();
return(null);
}
List <OVRLipSync.Frame> frames = new List <OVRLipSync.Frame>();
float[] samples = new float[sSampleSize * clip.channels];
OVRLipSync.Frame dummyFrame = new OVRLipSync.Frame();
OVRLipSync.ProcessFrame(
context,
samples,
dummyFrame,
(clip.channels == 2) ? true : false
);
// frame delay in ms
float frameDelayInMs = dummyFrame.frameDelay;
int frameOffset = (int)(frameDelayInMs * clip.frequency / 1000);
int totalSamples = clip.samples;
for (int x = 0; x < totalSamples + frameOffset; x += sSampleSize)
{
int remainingSamples = totalSamples - x;
if (remainingSamples >= sSampleSize)
{
clip.GetData(samples, x);
}
else if (remainingSamples > 0)
{
float[] samples_clip = new float[remainingSamples * clip.channels];
clip.GetData(samples_clip, x);
Array.Copy(samples_clip, samples, samples_clip.Length);
Array.Clear(samples, samples_clip.Length, samples.Length - samples_clip.Length);
}
else
{
Array.Clear(samples, 0, samples.Length);
}
OVRLipSync.Frame frame = new OVRLipSync.Frame();
if (clip.channels == 2)
{
// interleaved = stereo data, alternating floats
OVRLipSync.ProcessFrame(context, samples, frame);
}
else
{
// mono
OVRLipSync.ProcessFrame(context, samples, frame, false);
}
if (x < frameOffset)
{
continue;
}
frames.Add(frame);
}
Debug.Log(clip.name + " produced " + frames.Count +
" viseme frames, playback rate is " + (frames.Count / clip.length) +
" fps");
OVRLipSync.DestroyContext(context);
OVRLipSync.Shutdown();
sequence = ScriptableObject.CreateInstance <OVRLipSyncSequence>();
sequence.entries = frames;
sequence.length = clip.length;
return(sequence);
}
// Update is called once per frame
void Update()
{
if (EnableLipSync)
{
if (Context != 0)
{
if (proxy == null)
{
if (VRMmodel != null)
{
proxy = VRMmodel.GetComponent <VRMBlendShapeProxy>();
}
}
else
{
// get the current viseme frame
OVRLipSync.Frame frame = GetCurrentPhonemeFrame();
if (frame != null)
{
//あ OVRLipSync.Viseme.aa; BlendShapePreset.A;
//い OVRLipSync.Viseme.ih; BlendShapePreset.I;
//う OVRLipSync.Viseme.ou; BlendShapePreset.U;
//え OVRLipSync.Viseme.E; BlendShapePreset.E;
//お OVRLipSync.Viseme.oh; BlendShapePreset.O;
var presets = new BlendShapePreset[] {
BlendShapePreset.A,
BlendShapePreset.I,
BlendShapePreset.U,
BlendShapePreset.E,
BlendShapePreset.O,
};
var visemes = new float[] {
frame.Visemes[(int)OVRLipSync.Viseme.aa],
frame.Visemes[(int)OVRLipSync.Viseme.ih],
frame.Visemes[(int)OVRLipSync.Viseme.ou],
frame.Visemes[(int)OVRLipSync.Viseme.E],
frame.Visemes[(int)OVRLipSync.Viseme.oh],
};
int maxindex = 0;
float maxvisemes = 0;
for (int i = 0; i < presets.Length; i++)
{
if (visemes[i] < WeightThreashold)
{
visemes[i] = 0;
}
if (maxvisemes < visemes[i])
{
maxindex = i;
maxvisemes = visemes[i];
}
}
if (MaxWeightEmphasis)
{
visemes[maxindex] = Mathf.Clamp(visemes[maxindex] * 3, 0.0f, 1.0f);
}
if (MaxWeightEnable)
{
for (int i = 0; i < presets.Length; i++)
{
if (i != maxindex)
{
visemes[i] = 0.0f;
}
}
}
for (int i = 0; i < presets.Length; i++)
{
visemes[i] *= MaxLevel;
proxy.SetValue(presets[i], visemes[i]);
}
//Debug.Log("Visemes:" + string.Join(",", frame.Visemes.Select(d => d.ToString())));
}
}
}
if (string.IsNullOrEmpty(selectedDevice) == false)
{
audioSource.volume = (sourceVolume / 100);
if (!Microphone.IsRecording(selectedDevice))
{
StartMicrophone();
}
if (EnableLowLatency)
{
var position = Microphone.GetPosition(selectedDevice);
if (position < 0 || head == position)
{
return;
}
audioSource.clip.GetData(microphoneBuffer, 0);
while (GetDataLength(microphoneBuffer.Length, head, position) > processBuffer.Length)
{
var remain = microphoneBuffer.Length - head;
if (remain < processBuffer.Length)
{
Array.Copy(microphoneBuffer, head, processBuffer, 0, remain);
Array.Copy(microphoneBuffer, 0, processBuffer, remain, processBuffer.Length - remain);
}
else
{
Array.Copy(microphoneBuffer, head, processBuffer, 0, processBuffer.Length);
}
OVRLipSync.ProcessFrame(Context, processBuffer, Frame);
head += processBuffer.Length;
if (head > microphoneBuffer.Length)
{
head -= microphoneBuffer.Length;
}
}
}
}
}
}
public static OVRLipSyncSequence CreateSequenceFromAudioClip(AudioClip clip)
{
OVRLipSyncSequence sequence = null;
if (clip.loadType != AudioClipLoadType.DecompressOnLoad || clip.channels > 2)
{
// todo: just fix the clip
Debug.LogError("Cannot process phonemes from an audio clip unless its load type is set to DecompressOnLoad.");
}
else
{
if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success)
{
Debug.LogError("Could not create Lip Sync engine.");
}
else
{
uint context = 0;
OVRLipSync.Result result = OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Main);
if (result != OVRLipSync.Result.Success)
{
Debug.LogError("Could not create Phoneme context. (" + result + ")");
OVRLipSync.Shutdown();
}
else
{
List <OVRLipSync.Frame> frames = new List <OVRLipSync.Frame>();
float[] samples = new float[sSampleSize * clip.channels];
int totalSamples = clip.samples;
for (int x = 0; x < totalSamples; x += sSampleSize)
{
// GetData loops at the end of the read. Prevent that when it happens.
if (x + samples.Length > totalSamples)
{
samples = new float[(totalSamples - x) * clip.channels];
}
clip.GetData(samples, x);
OVRLipSync.Frame frame = new OVRLipSync.Frame();
if (clip.channels == 2)
{
// interleaved = stereo data, alternating floats
OVRLipSync.ProcessFrameInterleaved(context, samples, 0, frame);
}
else
{
// mono
OVRLipSync.ProcessFrame(context, samples, 0, frame);
}
frames.Add(frame);
}
Debug.Log(clip.name + " produced " + frames.Count + " viseme frames, playback rate is " + (frames.Count / clip.length) + " fps");
OVRLipSync.DestroyContext(context);
OVRLipSync.Shutdown();
sequence = ScriptableObject.CreateInstance <OVRLipSyncSequence>();
sequence.entries = frames;
sequence.length = clip.length;
}
}
}
return(sequence);
}
void ProcessBuffer(float[] buffer)
{
if (buffer == null)
{
return;
}
audioVolume = 0f;
foreach (float v in buffer)
{
audioVolume += Mathf.Abs(v);
}
audioVolume /= buffer.Length;
int totalLen = partialPos + buffer.Length;
int bufferPos = 0;
if (totalLen >= 1024 * channels)
{
volume = 0f;
while (totalLen >= 1024 * channels)
{
int sliceLen = 1024 - partialPos;
Array.Copy(buffer, bufferPos, partialAudio, partialPos, sliceLen * channels);
totalLen -= 1024 * channels;
if (totalLen < 1024 * channels)
{
for (int i = 0; i < partialAudio.Length; i++)
{
partialAudio[i] = partialAudio[i] * gain;//Mathf.Clamp(partialAudio[i] * gain, 0f, 1f);
volume += Mathf.Abs(partialAudio[i]);
}
lock (this) {
if (context != 0)
{
OVRLipSync.Frame frame = this.visemeData;
if (channels == 2)
{
OVRLipSync.ProcessFrameInterleaved(context, partialAudio, frame);
}
else
{
OVRLipSync.ProcessFrame(context, partialAudio, frame);
}
}
else
{
Debug.Log("OVRLipSync context is 0");
}
}
}
bufferPos += sliceLen;
partialPos = 0;
}
volume /= (float)buffer.Length;
}
if (totalLen > 0)
{
Array.Copy(buffer, bufferPos, partialAudio, partialPos, buffer.Length - bufferPos);
partialPos += (buffer.Length - bufferPos) / channels;
}
}
public static OVRLipSyncSequence CreateSequenceFromAudioClip(
AudioClip clip, bool useOfflineModel = false)
{
OVRLipSyncSequence sequence = null;
if (clip.channels > 2)
{
Debug.LogError(clip.name +
": Cannot process phonemes from an audio clip with " +
"more than 2 channels");
return(null);
}
if (clip.loadType != AudioClipLoadType.DecompressOnLoad)
{
Debug.LogError(clip.name +
": Cannot process phonemes from an audio clip unless " +
"its load type is set to DecompressOnLoad.");
return(null);
}
if (OVRLipSync.Initialize(clip.frequency, sSampleSize) != OVRLipSync.Result.Success)
{
Debug.LogError("Could not create Lip Sync engine.");
return(null);
}
if (clip.loadState != AudioDataLoadState.Loaded)
{
Debug.LogError("Clip is not loaded!");
return(null);
}
uint context = 0;
OVRLipSync.Result result = useOfflineModel
? OVRLipSync.CreateContextWithModelFile(
ref context,
OVRLipSync.ContextProviders.Enhanced,
Path.Combine(Application.dataPath, "Oculus/LipSync/Assets/OfflineModel/ovrlipsync_offline_model.pb"))
: OVRLipSync.CreateContext(ref context, OVRLipSync.ContextProviders.Enhanced);
if (result != OVRLipSync.Result.Success)
{
Debug.LogError("Could not create Phoneme context. (" + result + ")");
OVRLipSync.Shutdown();
return(null);
}
List <OVRLipSync.Frame> frames = new List <OVRLipSync.Frame>();
float[] samples = new float[sSampleSize * clip.channels];
int totalSamples = clip.samples;
for (int x = 0; x < totalSamples; x += sSampleSize)
{
// GetData loops at the end of the read. Prevent that when it happens.
if (x + samples.Length > totalSamples)
{
samples = new float[(totalSamples - x) * clip.channels];
}
clip.GetData(samples, x);
OVRLipSync.Frame frame = new OVRLipSync.Frame();
if (clip.channels == 2)
{
// interleaved = stereo data, alternating floats
OVRLipSync.ProcessFrame(context, samples, frame);
}
else
{
// mono
OVRLipSync.ProcessFrame(context, samples, frame, false);
}
frames.Add(frame);
}
Debug.Log(clip.name + " produced " + frames.Count +
" viseme frames, playback rate is " + (frames.Count / clip.length) +
" fps");
OVRLipSync.DestroyContext(context);
OVRLipSync.Shutdown();
sequence = ScriptableObject.CreateInstance <OVRLipSyncSequence>();
sequence.entries = frames;
sequence.length = clip.length;
return(sequence);
}