void Update()
{
// No sentence to analyze left (all done).
if (RecordingSentence == null)
{
return;
}
// Analyze the next beat with fully recorded mic samples
double nextBeatToAnalyzeEndPositionInMs = BpmUtils.BeatToMillisecondsInSong(songMeta, BeatToAnalyze + 1);
if (nextBeatToAnalyzeEndPositionInMs < songAudioPlayer.PositionInSongInMillis - micProfile.DelayInMillis)
{
// The beat has passed and should have recorded samples in the mic buffer. Analyze the samples now.
PitchEvent pitchEvent = GetPitchEventOfBeat(BeatToAnalyze);
Note currentOrUpcomingNote = currentAndUpcomingNotesInRecordingSentence.IsNullOrEmpty()
? null
: currentAndUpcomingNotesInRecordingSentence[0];
Note noteAtBeat = (currentOrUpcomingNote.StartBeat <= BeatToAnalyze && BeatToAnalyze < currentOrUpcomingNote.EndBeat)
? currentOrUpcomingNote
: null;
FirePitchEvent(pitchEvent, BeatToAnalyze, noteAtBeat, RecordingSentence);
GoToNextBeat();
}
}
private void HandlePitchEvent(PitchEvent pitchEvent, double currentBeat)
{
if (pitchEvent == null || pitchEvent.MidiNote <= 0)
{
if (lastRecordedNote != null)
{
HandleRecordedNoteEnded(currentBeat);
}
}
else
{
if (lastRecordedNote != null)
{
if (MidiUtils.GetRelativePitchDistance(lastRecordedNote.RoundedMidiNote, pitchEvent.MidiNote) <= roundingDistance)
{
// Continue singing on same pitch
HandleRecordedNoteContinued(currentBeat);
}
else
{
// Continue singing on different pitch. Finish the last recorded note.
HandleRecordedNoteEnded(currentBeat);
}
}
// The lastRecordedNote could be ended above, so the following null check is not redundant.
if (lastRecordedNote == null && currentBeat >= nextNoteStartBeat)
{
// Start singing of a new note
HandleRecordedNoteStarted(pitchEvent.MidiNote, currentBeat);
}
}
}
private void TestPitchDetection(Func <int, IAudioSamplesAnalyzer> audioSamplesAnalyzerProvider)
{
MicProfile micProfile = CreateDummyMicProfile();
string assetsPath = Application.dataPath;
string sineWaveToneDir = assetsPath + "/Common/Audio/SineWaveTones/";
Dictionary <string, string> pathToExpectedMidiNoteNameMap = new();
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-a3-220hz.ogg", "A3");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-a4-440hz.ogg", "A4");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-a5-880hz.ogg", "A5");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-c2-61,74hz.ogg", "C2");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-c4-261,64hz.ogg", "C4");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-c6-1046,50hz.ogg", "C6");
foreach (KeyValuePair <string, string> pathAndNoteName in pathToExpectedMidiNoteNameMap)
{
// Load the audio clip
string uri = "file://" + pathAndNoteName.Key;
AudioClip audioClip = AudioUtils.GetAudioClipUncached(uri, false);
float[] samples = new float[audioClip.samples];
audioClip.GetData(samples, 0);
// Analyze the samples
IAudioSamplesAnalyzer audioSamplesAnalyzer = audioSamplesAnalyzerProvider(audioClip.frequency);
PitchEvent pitchEvent = audioSamplesAnalyzer.ProcessAudioSamples(samples, 0, samples.Length - 1, micProfile);
// Check result
Assert.NotNull(pitchEvent, $"No pitch detected when analyzing audio resource {uri}");
string expectedName = pathAndNoteName.Value;
string analyzedName = MidiUtils.GetAbsoluteName(pitchEvent.MidiNote);
Assert.AreEqual(expectedName, analyzedName,
$"Expected {expectedName} but was {analyzedName} when analyzing audio resource {uri}");
}
}
protected override void OnRecordingEvent(RecordingEvent recordingEvent)
{
// Detect the pitch of the sample
int newSampleLength = recordingEvent.NewSampleCount;
bufferedNotAnalyzedSampleCount += newSampleLength;
// Wait until enough new samples are buffered
if (bufferedNotAnalyzedSampleCount < MinSampleCountToUse)
{
return;
}
// Do not analyze more than necessary
if (bufferedNotAnalyzedSampleCount > MaxSampleCountToUse)
{
bufferedNotAnalyzedSampleCount = MaxSampleCountToUse;
}
// Analyze the newest portion of the not-yet-analyzed MicSamples
int startIndex = recordingEvent.MicSamples.Length - bufferedNotAnalyzedSampleCount;
int endIndex = recordingEvent.MicSamples.Length;
PitchEvent pitchEvent = AudioSamplesAnalyzer.ProcessAudioSamples(recordingEvent.MicSamples, startIndex, endIndex, MicProfile);
bufferedNotAnalyzedSampleCount = 0;
// Notify listeners
pitchEventStream.OnNext(pitchEvent);
}
void Update()
{
// No sentence to analyze left (all done).
if (RecordingSentence == null)
{
return;
}
// Analyze the next beat with fully recorded mic samples
double nextBeatToAnalyzeEndPositionInMs = BpmUtils.BeatToMillisecondsInSong(songMeta, BeatToAnalyze + 1);
if (nextBeatToAnalyzeEndPositionInMs < songAudioPlayer.PositionInSongInMillis - micProfile.DelayInMillis)
{
// The beat has passed and should have recorded samples in the mic buffer. Analyze the samples now.
int startSampleBufferIndex = GetMicSampleBufferIndexForBeat(BeatToAnalyze);
int endSampleBufferIndex = GetMicSampleBufferIndexForBeat(BeatToAnalyze + 1);
if (startSampleBufferIndex > endSampleBufferIndex)
{
ObjectUtils.Swap(ref startSampleBufferIndex, ref endSampleBufferIndex);
}
Note currentOrUpcomingNote = currentAndUpcomingNotesInRecordingSentence[0];
Note noteAtBeat = (currentOrUpcomingNote.StartBeat <= BeatToAnalyze && BeatToAnalyze < currentOrUpcomingNote.EndBeat)
? currentOrUpcomingNote
: null;
PitchEvent pitchEvent = audioSamplesAnalyzer.ProcessAudioSamples(micSampleRecorder.MicSamples, startSampleBufferIndex, endSampleBufferIndex, micProfile);
FirePitchEvent(pitchEvent, BeatToAnalyze, noteAtBeat);
GoToNextBeat();
}
}
private int ApplyJokerRule(PitchEvent pitchEvent, int roundedMidiNote, Note noteAtBeat)
{
// Earn a joker when singing correctly (without using a joker).
// A failed beat can be undone via joker-rule.
if (pitchEvent != null && roundedMidiNote == noteAtBeat.MidiNote)
{
hasJoker = true;
}
// The joker is only for continued singing.
if (pitchEvent == null)
{
hasJoker = false;
}
// If the player fails a beat in continued singing, but the previous beats were sung correctly,
// then this failed beat is ignored.
if (roundedMidiNote != noteAtBeat.MidiNote &&
hasJoker)
{
hasJoker = false;
usedJokerCount++;
return(noteAtBeat.MidiNote);
}
return(roundedMidiNote);
}
public BeatAnalyzedEvent(PitchEvent pitchEvent, int beat, Note noteAtBeat, int roundedMidiNote)
{
PitchEvent = pitchEvent;
Beat = beat;
NoteAtBeat = noteAtBeat;
RoundedMidiNote = roundedMidiNote;
}
private void OnPitchDetected(PitchEvent pitchEvent)
{
if (pitchEvent == null || !isCalibrationInProgress || pauseTime > 0)
{
return;
}
string targetMidiNoteName = midiNoteNames[currentIteration];
if (MidiUtils.GetAbsoluteName(pitchEvent.MidiNote) == targetMidiNoteName)
{
audioSource.Stop();
float delayInSeconds = Time.time - startTimeInSeconds;
int delayInMillis = (int)(delayInSeconds * 1000);
delaysInMillis.Add(delayInMillis);
Debug.Log($"Mic delay calibration - delay of iteration {currentIteration}: {delayInMillis}");
currentIteration++;
if (currentIteration >= midiNoteNames.Count)
{
OnEndCalibration();
}
else
{
// Wait a bit for silence before the next iteration.
pauseTime = 0.5f;
}
}
else
{
Debug.Log("Mic delay calibration - wrong pitch: " + MidiUtils.GetAbsoluteName(pitchEvent.MidiNote));
}
}
public PitchEvent GetPitchEventOfBeat(int beat)
{
int startSampleBufferIndex = GetMicSampleBufferIndexForBeat(beat);
int endSampleBufferIndex = GetMicSampleBufferIndexForBeat(beat + 1);
PitchEvent pitchEvent = GetPitchEventOfSamples(startSampleBufferIndex, endSampleBufferIndex);
return(pitchEvent);
}
protected override PitchEvent GetDummyPichtEvent(int beat, Note noteAtBeat)
{
PitchEvent pitchEvent = null;
if (noteAtBeat != null && Random.Range(0, 5) != 0)
{
pitchEvent = new PitchEvent(noteAtBeat.MidiNote + Random.Range(-3, 3));
}
return(pitchEvent);
}
public void FirePitchEvent(PitchEvent pitchEvent, int beat, Note noteAtBeat)
{
int roundedMidiNote = pitchEvent != null
? GetRoundedMidiNoteForRecordedMidiNote(noteAtBeat, pitchEvent.MidiNote)
: -1;
int roundedMidiNoteAfterJoker = ApplyJokerRule(pitchEvent, roundedMidiNote, noteAtBeat);
beatAnalyzedEventStream.OnNext(new BeatAnalyzedEvent(pitchEvent, beat, noteAtBeat, roundedMidiNoteAfterJoker));
}
protected override PitchEvent GetDummyPichtEvent(int beat, Note noteAtBeat)
{
PitchEvent pitchEvent = null;
if (noteAtBeat != null)
{
pitchEvent = new PitchEvent(noteAtBeat.MidiNote + offset);
}
return(pitchEvent);
}
public override void UpdateSinging(double currentBeat)
{
Note noteAtCurrentBeat = GetNoteAtCurrentBeat(currentBeat);
PitchEvent pitchEvent = null;
if (noteAtCurrentBeat != null)
{
pitchEvent = new PitchEvent(noteAtCurrentBeat.MidiNote + offset);
}
playerController.PlayerNoteRecorder.HandlePitchEvent(pitchEvent);
}
public override void UpdateSinging(double currentBeat)
{
Note noteAtCurrentBeat = GetNoteAtCurrentBeat(currentBeat);
PitchEvent pitchEvent = null;
if (noteAtCurrentBeat != null && Random.Range(0, 5) != 0)
{
pitchEvent = new PitchEvent(noteAtCurrentBeat.MidiNote + Random.Range(-3, 3));
}
playerController.PlayerNoteRecorder.HandlePitchEvent(pitchEvent);
}
public PitchEvent GetPitchEventOfSamples(int startSampleBufferIndex, int endSampleBufferIndex)
{
if (startSampleBufferIndex > endSampleBufferIndex)
{
ObjectUtils.Swap(ref startSampleBufferIndex, ref endSampleBufferIndex);
}
startSampleBufferIndex = NumberUtils.Limit(startSampleBufferIndex, 0, micSampleRecorder.MicSamples.Length - 1);
endSampleBufferIndex = NumberUtils.Limit(endSampleBufferIndex, 0, micSampleRecorder.MicSamples.Length - 1);
PitchEvent pitchEvent = audioSamplesAnalyzer.ProcessAudioSamples(micSampleRecorder.MicSamples, startSampleBufferIndex, endSampleBufferIndex, micProfile);
return(pitchEvent);
}
private void OnPitchDetected(PitchEvent pitchEvent)
{
if (pitchEvent == null || lastPitchDetectedFrame == Time.frameCount)
{
return;
}
double positionInSongInMillis = songAudioPlayer.PositionInSongInMillis - EditorSettings.MicDelayInMillis;
int midiNote = pitchEvent.MidiNote + (EditorSettings.MicOctaveOffset * 12);
RecordNote(midiNote, positionInSongInMillis, ESongEditorLayer.MicRecording);
}
private void OnPitchDetected(PitchEvent pitchEvent)
{
// Show the note that has been detected
if (pitchEvent != null && pitchEvent.MidiNote > 0)
{
currentNoteLabel.text = "Note: " + MidiUtils.GetAbsoluteName(pitchEvent.MidiNote);
}
else
{
currentNoteLabel.text = "Note: ?";
}
}
private void UpdateLastMidiNoteFields(PitchEvent pitchEvent)
{
if (pitchEvent == null)
{
lastMidiNoteName = "";
lastMidiNote = 0;
}
else
{
lastMidiNoteName = MidiUtils.GetAbsoluteName(pitchEvent.MidiNote);
lastMidiNote = pitchEvent.MidiNote;
}
}
private void OnPitchDetected(PitchEvent pitchEvent)
{
// Show the note that has been detected
if (pitchEvent != null && pitchEvent.MidiNote > 0)
{
currentNoteLabel.text = TranslationManager.GetTranslation(R.Messages.options_note,
"value", MidiUtils.GetAbsoluteName(pitchEvent.MidiNote));
}
else
{
currentNoteLabel.text = TranslationManager.GetTranslation(R.Messages.options_note,
"value", "?");
}
}
public BeatAnalyzedEvent(PitchEvent pitchEvent,
int beat,
Note noteAtBeat,
Sentence sentenceAtBeat,
int recordedMidiNote,
int roundedRecordedMidiNote)
{
PitchEvent = pitchEvent;
Beat = beat;
NoteAtBeat = noteAtBeat;
SentenceAtBeat = sentenceAtBeat;
RecordedMidiNote = recordedMidiNote;
RoundedRecordedMidiNote = roundedRecordedMidiNote;
}
public async Task <IActionResult> OnGetAsync(int?id)
{
if (id == null)
{
return(NotFound());
}
PitchEvent = await _context.PitchEvents.FirstOrDefaultAsync(m => m.PitchEventId == id);
if (PitchEvent == null)
{
return(NotFound());
}
return(Page());
}
void Update()
{
int currentBeat = (int)singSceneController.CurrentBeat;
int beatToAnalyze = playerController.PlayerPitchTracker.BeatToAnalyze;
if (currentBeat > 0 &&
beatToAnalyze <= currentBeat &&
playerController.PlayerPitchTracker.RecordingSentence != null)
{
Note noteAtBeat = GetNoteAtBeat(beatToAnalyze);
PitchEvent pitchEvent = GetDummyPichtEvent(beatToAnalyze, noteAtBeat);
playerController.PlayerPitchTracker.FirePitchEvent(pitchEvent, beatToAnalyze, noteAtBeat);
playerController.PlayerPitchTracker.GoToNextBeat();
}
}
public async Task <IActionResult> OnPostAsync(int?id)
{
if (id == null)
{
return(NotFound());
}
PitchEvent = await _context.PitchEvents.FindAsync(id);
if (PitchEvent != null)
{
_context.PitchEvents.Remove(PitchEvent);
await _context.SaveChangesAsync();
}
return(RedirectToPage("./Index"));
}
protected override PitchEvent GetDummyPichtEvent(int beat, Note noteAtBeat)
{
PitchEvent pitchEvent = null;
if (noteAtBeat != null)
{
pitchEvent = new PitchEvent(noteAtBeat.MidiNote + noteOffset);
}
// Change noteOffset when note changes.
if (lastNote != null && noteAtBeat != lastNote)
{
noteOffset = (noteOffset + 1) % maxOffset;
}
lastNote = noteAtBeat;
return(pitchEvent);
}
void Update()
{
if (songAudioPlayer.PositionInSongInMillis <= 0)
{
return;
}
PitchEvent pitchEvent = playerPitchTracker.GetPitchEventOfSamples(micSampleRecorder.MicSamples.Length - 1 - samplesPerBeat, micSampleRecorder.MicSamples.Length - 1);
if (pitchEvent != null)
{
// Shift midi note to octave of last recorded midi note (can be different because PlayerPitchTracker is rounding towards the target note)
int midiNote = lastRecordedRoundedMidiNote > 0
? MidiUtils.GetRelativePitch(pitchEvent.MidiNote) + (12 * (MidiUtils.GetOctave(lastRecordedRoundedMidiNote) + 1))
: pitchEvent.MidiNote;
UpdatePosition(midiNote);
}
}
public void HandlePitchEvent(PitchEvent pitchEvent)
{
double currentBeat = GetCurrentBeat();
// It could be that some beats have been missed, for example because the frame rate was too low.
// In this case, the pitch event is fired here also for the missed beats.
if (lastBeatWithPitchEvent < currentBeat)
{
int missedBeats = (int)(currentBeat - lastBeatWithPitchEvent);
for (int i = 1; i <= missedBeats; i++)
{
HandlePitchEvent(pitchEvent, lastBeatWithPitchEvent + i);
}
}
HandlePitchEvent(pitchEvent, currentBeat);
lastBeatWithPitchEvent = currentBeat;
}
public override void UpdateSinging(double currentBeat)
{
Note noteAtCurrentBeat = GetNoteAtCurrentBeat(currentBeat);
PitchEvent pitchEvent = null;
if (noteAtCurrentBeat != null)
{
pitchEvent = new PitchEvent(noteAtCurrentBeat.MidiNote + noteOffset);
}
else if (lastNote != null)
{
// Change noteOffset on falling flank of note
// (falling flank: now there is no note, but in last frame there was one)
noteOffset = (noteOffset + 1) % 5;
}
playerController.PlayerNoteRecorder.HandlePitchEvent(pitchEvent);
lastNote = noteAtCurrentBeat;
}
protected override PitchEvent GetDummyPichtEvent(int beat, Note noteAtBeat)
{
// Change noteOffset when note changes.
if (lastNote != null && noteAtBeat != lastNote)
{
noteOffset = (noteOffset + 1) % maxOffset;
}
PitchEvent pitchEvent = null;
if (noteAtBeat != null)
{
int dummyMidiNote = noteAtBeat.MidiNote + noteOffset;
int dummyMidiNoteInSingableNoteRange = (dummyMidiNote % 12) + (5 * 12);
pitchEvent = new PitchEvent(dummyMidiNoteInSingableNoteRange);
}
lastNote = noteAtBeat;
return(pitchEvent);
}
private void AnalyzeBeatAndNotify(int beat)
{
PitchEvent pitchEvent = AnalyzeBeat(
songMeta,
beat,
songAudioPlayer.PositionInSongInMillis,
MicSampleRecorder.MicProfile,
MicSampleRecorder.MicSamples,
AudioSamplesAnalyzer);
// Notify listeners
if (pitchEvent == null)
{
pitchEventStream.OnNext(null);
}
else
{
int shiftedMidiNote = pitchEvent.MidiNote + (songEditorSettings.SongEditorSettings.MicOctaveOffset * 12);
pitchEventStream.OnNext(new BeatPitchEvent(shiftedMidiNote, beat));
}
}
public void TestPitchDetection()
{
MicProfile micProfile = CreateDummyMicProfile();
string assetsPath = Application.dataPath;
string sineWaveToneDir = assetsPath + "/Editor/Tests/SineWaveTones/";
Dictionary <string, string> pathToExpectedMidiNoteNameMap = new Dictionary <string, string>();
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-a3-220hz.ogg", "A3");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-a4-440hz.ogg", "A4");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-a5-880hz.ogg", "A5");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-c2-61,74hz.ogg", "C2");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-c4-261,64hz.ogg", "C4");
pathToExpectedMidiNoteNameMap.Add(sineWaveToneDir + "sine-wave-c6-1046,50hz.ogg", "C6");
foreach (KeyValuePair <string, string> pathAndNoteName in pathToExpectedMidiNoteNameMap)
{
// Load the audio clip
string path = pathAndNoteName.Key;
AudioClip audioClip = AudioUtils.GetAudioClip(path);
float[] samples = new float[audioClip.samples];
audioClip.GetData(samples, 0);
// Analyze the samples
IAudioSamplesAnalyzer audioSamplesAnalyzer = new CamdAudioSamplesAnalyzer(audioClip.frequency);
audioSamplesAnalyzer.Enable();
PitchEvent pitchEvent = audioSamplesAnalyzer.ProcessAudioSamples(samples, samples.Length, micProfile);
// Check result
Assert.NotNull(pitchEvent, $"No pitch detected when analyzing {path}");
string expectedName = pathAndNoteName.Value;
string analyzedName = MidiUtils.GetAbsoluteName(pitchEvent.MidiNote);
Assert.AreEqual(expectedName, analyzedName,
$"Expected {expectedName} but was {analyzedName} when analyzing {path}");
}
}