public void AdjustAndInterpolate(SpeechTypeSpecification speechTypeSpecification) // , double deltaTime, Boolean setUnvoicedPitch)
{
// Carry out median filtering to remove single errors
List <double> correctedPitchValues = new List <double>();
correctedPitchValues.Add(pitchPeriodSpecification.TimePitchPeriodTupleList[0].Item2);
for (int ii = 1; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count - 1; ii++)
{
List <double> rawPitchValues = new List <double>()
{
pitchPeriodSpecification.TimePitchPeriodTupleList[ii - 1].Item2,
pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item2,
pitchPeriodSpecification.TimePitchPeriodTupleList[ii + 1].Item2
};
rawPitchValues.Sort();
correctedPitchValues.Add(rawPitchValues[1]); // Median
}
// Finally adjust the end points (which are not touched by the initial median filtering)
if (pitchPeriodSpecification.TimePitchPeriodTupleList.Count > 2)
{
List <double> rawPitchValues = new List <double>()
{
pitchPeriodSpecification.TimePitchPeriodTupleList[0].Item2,
pitchPeriodSpecification.TimePitchPeriodTupleList[1].Item2,
pitchPeriodSpecification.TimePitchPeriodTupleList[2].Item2
};
rawPitchValues.Sort();
correctedPitchValues[0] = rawPitchValues[1];
int lastIndex = pitchPeriodSpecification.TimePitchPeriodTupleList.Count - 1;
rawPitchValues = new List <double>()
{
pitchPeriodSpecification.TimePitchPeriodTupleList[lastIndex].Item2,
pitchPeriodSpecification.TimePitchPeriodTupleList[lastIndex - 1].Item2,
pitchPeriodSpecification.TimePitchPeriodTupleList[lastIndex - 2].Item2
};
rawPitchValues.Sort();
correctedPitchValues.Add(rawPitchValues[1]);
}
for (int ii = 0; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count - 1; ii++)
{
pitchPeriodSpecification.TimePitchPeriodTupleList[ii] =
new Tuple <double, double>(pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item1,
correctedPitchValues[ii]);
}
// Extend (extrapolate) the pitch period specification so that it runs to the end of the sound:
double lastTime = speechTypeSpecification.TimeSpeechTypeTupleList.Last().Item1;
int lastPitchIndex = pitchPeriodSpecification.TimePitchPeriodTupleList.Count - 1;
double lastPitchTime = pitchPeriodSpecification.TimePitchPeriodTupleList[lastPitchIndex].Item1;
if (lastTime > lastPitchTime) // Should always be the case, but just to be sure ...
{
double lastPitch = pitchPeriodSpecification.TimePitchPeriodTupleList[lastPitchIndex].Item2;
pitchPeriodSpecification.TimePitchPeriodTupleList.Add(new Tuple <double, double>(lastTime, lastPitch));
}
// Next, resample (upsample) the pitch period specification
List <double> timeList = new List <double>();
List <double> pitchList = new List <double>();
for (int ii = 0; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; ii++)
{
double time = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item1;
double pitch = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item2;
timeList.Add(time);
pitchList.Add(pitch);
}
List <List <double> > timePitchList = new List <List <double> >()
{
timeList, pitchList
};
int numberOfPoints = (int)Math.Round(lastTime / deltaTime);
List <List <double> > interpolatedTimePitchList = LinearInterpolation.Interpolate(timePitchList, numberOfPoints);
pitchPeriodSpecification = new PitchPeriodSpecification();
for (int ii = 0; ii < interpolatedTimePitchList[0].Count; ii++)
{
double time = interpolatedTimePitchList[0][ii];
double pitch = interpolatedTimePitchList[1][ii];
pitchPeriodSpecification.TimePitchPeriodTupleList.Add(new Tuple <double, double>(time, pitch));
}
// Optionally (usually true) hard-set the (anyway rather arbitrary) pitch period for
// unvoiced parts of the sound, by extending the pitch period from surrounding
// voiced parts. This might cause occasional jumps (in the middle of an unvoiced
// section), but those jumps are reoved in the subsequent lowpass filtering
if (setUnvoicedPitch)
{
double previousTime = pitchPeriodSpecification.TimePitchPeriodTupleList[0].Item1;
SpeechType previousSpeechType = speechTypeSpecification.GetSpeechType(previousTime);
int firstChangeIndex = 0; // Will be changed later - must initialize here.
int lastChangeIndex = -1;
double previousPitch; // Must define here for use after the loop as well.
for (int ii = 1; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; ii++)
{
double time = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item1;
SpeechType speechType = speechTypeSpecification.GetSpeechType(time);
if ((previousSpeechType == SpeechType.Voiced) && (speechType != SpeechType.Voiced))
{
firstChangeIndex = ii;
lastChangeIndex = -1; // Not yet assigned. The value -1 is used for handling cases where the
// sound remains not voiced until the end (see below).
}
else if ((previousSpeechType != SpeechType.Voiced) && (speechType == SpeechType.Voiced))
{
lastChangeIndex = ii - 1;
int middlexIndex = (firstChangeIndex + lastChangeIndex) / 2; // integer division
// assign the preceding pitch to the first half of the interval (unless firstChangeIndex = 0, meaning
// that the sound started with an unvoiced segment), and the subsequent pitch to the second half of
// the interval:
double subsequentPitch = pitchPeriodSpecification.TimePitchPeriodTupleList[lastChangeIndex].Item2;
previousPitch = subsequentPitch;
if (firstChangeIndex > 0)
{
previousPitch = pitchPeriodSpecification.TimePitchPeriodTupleList[firstChangeIndex - 1].Item2;
}
for (int jj = firstChangeIndex; jj < middlexIndex; jj++)
{
time = pitchPeriodSpecification.TimePitchPeriodTupleList[jj].Item1;
pitchPeriodSpecification.TimePitchPeriodTupleList[jj] = new Tuple <double, double>(time, previousPitch);
}
for (int jj = middlexIndex; jj <= lastChangeIndex; jj++)
{
time = pitchPeriodSpecification.TimePitchPeriodTupleList[jj].Item1;
pitchPeriodSpecification.TimePitchPeriodTupleList[jj] = new Tuple <double, double>(time, subsequentPitch);
}
}
previousTime = time;
previousSpeechType = speechType;
}
// At the end, if lastChangeIndex = -1, then the sound remained not voiced from the latest
// change until the end. Thus:
if ((lastChangeIndex == -1) && (firstChangeIndex > 0))
{
previousPitch = pitchPeriodSpecification.TimePitchPeriodTupleList[firstChangeIndex - 1].Item2;
for (int jj = firstChangeIndex; jj < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; jj++)
{
double time = pitchPeriodSpecification.TimePitchPeriodTupleList[jj].Item1;
pitchPeriodSpecification.TimePitchPeriodTupleList[jj] = new Tuple <double, double>(time, previousPitch);
}
}
// Then, finally, low-pass filter the interpolated list, and assign the result:
AveragingFilter averagingFilter = new AveragingFilter();
List <double> inputList = new List <double>();
for (int ii = 0; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; ii++)
{
double input = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item2;
inputList.Add(input);
}
List <double> outputList = averagingFilter.Run(inputList);
for (int ii = 0; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; ii++)
{
double filteredPitch = outputList[ii];
double time = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item1;
pitchPeriodSpecification.TimePitchPeriodTupleList[ii] = new Tuple <double, double>(time, filteredPitch);
}
/* FirstOrderLowPassFilter lowPassFilter = new FirstOrderLowPassFilter();
* lowPassFilter.SetAlpha(0.9); // To do: Parameterize.
* for (int ii = 0; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; ii++)
* {
* double input = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item2;
* lowPassFilter.Step(input);
* }
* for (int ii = 0; ii < pitchPeriodSpecification.TimePitchPeriodTupleList.Count; ii++)
* {
* double filteredPitch = lowPassFilter.OutputList[ii];
* double time = pitchPeriodSpecification.TimePitchPeriodTupleList[ii].Item1;
* pitchPeriodSpecification.TimePitchPeriodTupleList[ii] = new Tuple<double, double>(time, filteredPitch);
* } */
}
}
public void FindPitchMarks(WAVSound sound, SpeechTypeSpecification speechTypeSpecification, PitchPeriodSpecification pitchPeriodSpecification)
// , double peakSearchTimeRange,
// double adjustmentTimeRange, double relativePeakThreshold, double energyComputationTimeRange)
{
List <Tuple <int, int, SpeechType> > segmentTypeList = speechTypeSpecification.GetSegmentTypes();
List <int> absoluteSampleList = sound.GetAbsoluteSamples(0);
pitchMarkTimeList = new List <double>();
for (int iSegment = 0; iSegment < segmentTypeList.Count; iSegment++)
{
SpeechType segmentType = segmentTypeList[iSegment].Item3;
if (segmentType == SpeechType.Voiced)
{
int startIndex = segmentTypeList[iSegment].Item1;
int endIndex = segmentTypeList[iSegment].Item2;
double startTime = speechTypeSpecification.TimeSpeechTypeTupleList[startIndex].Item1;
double endTime = speechTypeSpecification.TimeSpeechTypeTupleList[endIndex].Item1;
int startSearchIndex = sound.GetSampleIndexAtTime(startTime);
int endSearchIndex = sound.GetSampleIndexAtTime(endTime);
int peakIndexSearchRange = (int)Math.Round(peakSearchTimeRange * sound.SampleRate);
int adjustmentIndexRange = (int)Math.Round(adjustmentTimeRange * sound.SampleRate);
int indexOfAbsoluteMaximum = sound.GetIndexOfAbsoluteMaximum(startSearchIndex, endSearchIndex);
int adjustedMainPitchMarkIndex = AdjustPitchMark(sound, indexOfAbsoluteMaximum, adjustmentIndexRange); // , relativePeakThreshold, energyComputationTimeRange);
double adjustedMainPitchMarkTime = sound.GetTimeAtSampleIndex(adjustedMainPitchMarkIndex);
pitchMarkTimeList.Add(adjustedMainPitchMarkTime);
Boolean inVoicedSegment = true;
double previousPitchMarkTime = adjustedMainPitchMarkTime;
// Next, move forward until the end of the voiced segment
while (inVoicedSegment)
{
double pitchPeriod = pitchPeriodSpecification.GetPitchPeriod(previousPitchMarkTime);
int deltaSample = (int)Math.Round(pitchPeriod * sound.SampleRate);
int previousSampleIndex = sound.GetSampleIndexAtTime(previousPitchMarkTime);
int pitchSampleIndex = previousSampleIndex + deltaSample;
if (pitchSampleIndex + 2 * peakIndexSearchRange >= sound.Samples[0].Count)
{
break;
}
int currentSampleIndex = sound.GetIndexOfAbsoluteMaximum(pitchSampleIndex - peakIndexSearchRange, pitchSampleIndex + peakIndexSearchRange);
double currentTime = sound.GetTimeAtSampleIndex(currentSampleIndex);
int adjustedSampleIndex = AdjustPitchMark(sound, currentSampleIndex, adjustmentIndexRange); //, relativePeakThreshold, energyComputationTimeRange);
if (adjustedSampleIndex <= previousSampleIndex)
{
adjustedSampleIndex = currentSampleIndex; // Emergency fallback in cases where the search gets stuck (can happen if the pitch period is too small relative to the search range)
}
double adjustedTime = sound.GetTimeAtSampleIndex(adjustedSampleIndex);
// Make an incursion into the non-voiced segment
pitchMarkTimeList.Add(adjustedTime);
previousPitchMarkTime = adjustedTime;
if (speechTypeSpecification.GetSpeechType(currentTime) != SpeechType.Voiced)
{
inVoicedSegment = false;
}
/* if (speechTypeSpecification.GetSpeechType(currentTime) == SpeechType.Voiced)
* {
* pitchMarkTimeList.Add(adjustedTime);
* previousPitchMarkTime = adjustedTime;
* }
* else { inVoicedSegment = false; } */
}
double voicedEndTime = pitchMarkTimeList.Last();
// Then continue half-way through any non-voiced segment followed by another voiced segment,
// or until the end of the sound if no voiced segment follows:
if (iSegment < segmentTypeList.Count)
{
double subsequenceVoicedSegmentStartTime = 0;
Boolean hasSubsequentVoicedSegment = false;
if (iSegment + 1 < segmentTypeList.Count)
{
for (int kk = iSegment + 1; kk < segmentTypeList.Count; kk++)
{
if (segmentTypeList[kk].Item3 == SpeechType.Voiced)
{
hasSubsequentVoicedSegment = true;
int startSegmentIndex = segmentTypeList[kk].Item1;
subsequenceVoicedSegmentStartTime = speechTypeSpecification.TimeSpeechTypeTupleList[startSegmentIndex].Item1;
break;
}
}
}
if (!hasSubsequentVoicedSegment)
{
// No following voiced segment: Just continue to the end
Boolean endReached = false;
while (!endReached)
{
double pitchPeriod = pitchPeriodSpecification.GetPitchPeriod(previousPitchMarkTime);
int deltaSample = (int)Math.Round(pitchPeriod * sound.SampleRate);
int previousSampleIndex = sound.GetSampleIndexAtTime(previousPitchMarkTime);
int pitchSampleIndex = previousSampleIndex + deltaSample;
if (pitchSampleIndex + 2 * peakIndexSearchRange >= sound.Samples[0].Count)
{
endReached = true;
break;
}
int currentSampleIndex = sound.GetIndexOfAbsoluteMaximum(pitchSampleIndex - peakIndexSearchRange, pitchSampleIndex + peakIndexSearchRange);
double currentTime = sound.GetTimeAtSampleIndex(currentSampleIndex);
int adjustedSampleIndex = AdjustPitchMark(sound, currentSampleIndex, adjustmentIndexRange); //, relativePeakThreshold, energyComputationTimeRange);
if (adjustedSampleIndex <= previousSampleIndex)
{
adjustedSampleIndex = currentSampleIndex; // Emergency fallback in cases where the search gets stuck (can happen if the pitch period is too small relative to the search range)
}
double adjustedTime = sound.GetTimeAtSampleIndex(adjustedSampleIndex);
pitchMarkTimeList.Add(adjustedTime);
previousPitchMarkTime = adjustedTime;
}
}
else // Proceed to the half-way mark of the interval from the end of the current voice segment to the beginning of the next.
{
double stopTime = voicedEndTime + (subsequenceVoicedSegmentStartTime - voicedEndTime) / 2;
int stopTimeIndex = sound.GetSampleIndexAtTime(stopTime);
Boolean endReached = false;
while (!endReached)
{
double pitchPeriod = pitchPeriodSpecification.GetPitchPeriod(previousPitchMarkTime);
int deltaSample = (int)Math.Round(pitchPeriod * sound.SampleRate);
int previousSampleIndex = sound.GetSampleIndexAtTime(previousPitchMarkTime);
int pitchSampleIndex = previousSampleIndex + deltaSample;
if (pitchSampleIndex + 2 * peakIndexSearchRange >= stopTimeIndex)
{
endReached = true;
break;
}
int currentSampleIndex = sound.GetIndexOfAbsoluteMaximum(pitchSampleIndex - peakIndexSearchRange, pitchSampleIndex + peakIndexSearchRange);
double currentTime = sound.GetTimeAtSampleIndex(currentSampleIndex);
int adjustedSampleIndex = AdjustPitchMark(sound, currentSampleIndex, adjustmentIndexRange); // , relativePeakThreshold, energyComputationTimeRange);
if (adjustedSampleIndex <= previousSampleIndex)
{
adjustedSampleIndex = currentSampleIndex; // Emergency fallback in cases where the search gets stuck (can happen if the pitch period is too small relative to the search range)
}
double adjustedTime = sound.GetTimeAtSampleIndex(adjustedSampleIndex);
pitchMarkTimeList.Add(adjustedTime);
previousPitchMarkTime = adjustedTime;
}
}
}
// Then move backward until the beginning of the voiced segment
inVoicedSegment = true;
previousPitchMarkTime = adjustedMainPitchMarkTime;
double voicedStartTime = 0;
while (inVoicedSegment)
{
double pitch = pitchPeriodSpecification.GetPitchPeriod(previousPitchMarkTime);
int deltaSample = -(int)Math.Round(pitch * sound.SampleRate);
int previousSampleIndex = sound.GetSampleIndexAtTime(previousPitchMarkTime);
int pitchSampleIndex = previousSampleIndex + deltaSample;
if (pitchSampleIndex - 2 * peakIndexSearchRange < 0)
{
break;
}
int currentSampleIndex = sound.GetIndexOfAbsoluteMaximum(pitchSampleIndex - peakIndexSearchRange, pitchSampleIndex + peakIndexSearchRange);
double currentTime = sound.GetTimeAtSampleIndex(currentSampleIndex);
int adjustedSampleIndex = AdjustPitchMark(sound, currentSampleIndex, peakIndexSearchRange); // , relativePeakThreshold, energyComputationTimeRange);
if (adjustedSampleIndex <= previousSampleIndex)
{
adjustedSampleIndex = currentSampleIndex; // Emergency fallback in cases where the search gets stuck (can happen if the pitch period is too small relative to the search range)
}
double adjustedTime = sound.GetTimeAtSampleIndex(adjustedSampleIndex);
// Make an incursion into the non-voiced segment
pitchMarkTimeList.Add(adjustedTime);
previousPitchMarkTime = adjustedTime;
if (speechTypeSpecification.GetSpeechType(currentTime) != SpeechType.Voiced)
{
inVoicedSegment = false;
voicedStartTime = adjustedTime;
}
}
// Then continue half-way through any non-voiced segment preceded by another voiced segment,
// or until the beginning of the sound if no voiced segment follows:
if (iSegment > 0)
{
double priorVoicedSegmentEndTime = 0;
Boolean hasPriorVoicedSegment = false;
if (iSegment - 1 > 0)
{
for (int kk = iSegment - 1; kk >= 0; kk--)
{
if (segmentTypeList[kk].Item3 == SpeechType.Voiced)
{
hasPriorVoicedSegment = true;
int endSegmentIndex = segmentTypeList[kk].Item2;
priorVoicedSegmentEndTime = speechTypeSpecification.TimeSpeechTypeTupleList[endSegmentIndex].Item1;
break;
}
}
}
if (!hasPriorVoicedSegment)
{
// No following voiced segment: Just continue to the end
Boolean endReached = false;
while (!endReached)
{
double pitchPeriod = pitchPeriodSpecification.GetPitchPeriod(previousPitchMarkTime);
int deltaSample = -(int)Math.Round(pitchPeriod * sound.SampleRate);
int previousSampleIndex = sound.GetSampleIndexAtTime(previousPitchMarkTime);
int pitchSampleIndex = previousSampleIndex + deltaSample;
if (pitchSampleIndex - 2 * peakIndexSearchRange < 0)
{
endReached = true;
break;
}
int currentSampleIndex = sound.GetIndexOfAbsoluteMaximum(pitchSampleIndex - peakIndexSearchRange, pitchSampleIndex + peakIndexSearchRange);
double currentTime = sound.GetTimeAtSampleIndex(currentSampleIndex);
int adjustedSampleIndex = AdjustPitchMark(sound, currentSampleIndex, adjustmentIndexRange); // , relativePeakThreshold, energyComputationTimeRange);
if (adjustedSampleIndex <= previousSampleIndex)
{
adjustedSampleIndex = currentSampleIndex; // Emergency fallback in cases where the search gets stuck (can happen if the pitch period is too small relative to the search range)
}
double adjustedTime = sound.GetTimeAtSampleIndex(adjustedSampleIndex);
pitchMarkTimeList.Add(adjustedTime);
previousPitchMarkTime = adjustedTime;
}
}
else // Proceed to the half-way mark of the interval from the end of the current voice segment to the beginning of the next.
{
double stopTime = voicedStartTime - (voicedStartTime - priorVoicedSegmentEndTime) / 2;
int stopTimeIndex = sound.GetSampleIndexAtTime(stopTime);
Boolean endReached = false;
while (!endReached)
{
double pitchPeriod = pitchPeriodSpecification.GetPitchPeriod(previousPitchMarkTime);
int deltaSample = -(int)Math.Round(pitchPeriod * sound.SampleRate);
int previousSampleIndex = sound.GetSampleIndexAtTime(previousPitchMarkTime);
int pitchSampleIndex = previousSampleIndex + deltaSample;
if (pitchSampleIndex - 2 * peakIndexSearchRange <= stopTimeIndex)
{
endReached = true;
break;
}
int currentSampleIndex = sound.GetIndexOfAbsoluteMaximum(pitchSampleIndex - peakIndexSearchRange, pitchSampleIndex + peakIndexSearchRange);
double currentTime = sound.GetTimeAtSampleIndex(currentSampleIndex);
int adjustedSampleIndex = AdjustPitchMark(sound, currentSampleIndex, adjustmentIndexRange); // , relativePeakThreshold, energyComputationTimeRange);
if (adjustedSampleIndex <= previousSampleIndex)
{
adjustedSampleIndex = currentSampleIndex; // Emergency fallback in cases where the search gets stuck (can happen if the pitch period is too small relative to the search range)
}
double adjustedTime = sound.GetTimeAtSampleIndex(adjustedSampleIndex);
pitchMarkTimeList.Add(adjustedTime);
previousPitchMarkTime = adjustedTime;
}
}
}
}
}
pitchMarkTimeList.Sort();
// Finally, remove any pitch marks that are too close (should only happen in non-voiced segments)
double minimumPitchPeriod = pitchPeriodSpecification.GetMinimumPitchPeriod();
int index = 1;
while (index < pitchMarkTimeList.Count)
{
double previousTime = pitchMarkTimeList[index - 1];
double currentTime = pitchMarkTimeList[index];
SpeechType previousSpeechType = speechTypeSpecification.GetSpeechType(previousTime);
SpeechType currentSpeechType = speechTypeSpecification.GetSpeechType(currentTime);
if ((previousSpeechType != SpeechType.Voiced) && (currentSpeechType != SpeechType.Voiced))
{
double deltaTime = currentTime - previousTime;
if (deltaTime < MINIMUM_UNVOICED_PITCHMARK_SPACING)
{
pitchMarkTimeList.RemoveAt(index);
}
else
{
index++;
}
}
else
{
index++;
}
}
}