private void TimeWarp(TimeWarpType type, AudioTrack t1, TimeSpan t1From, TimeSpan t1To, AudioTrack t2, TimeSpan t2From, TimeSpan t2To, bool calculateSimilarity, bool normalizeSimilarity, bool cueIn, bool cueOut) { IAudioStream s1 = t1.CreateAudioStream(); IAudioStream s2 = t2.CreateAudioStream(); s1 = new CropStream(s1, TimeUtil.TimeSpanToBytes(t1From, s1.Properties), TimeUtil.TimeSpanToBytes(t1To, s1.Properties)); s2 = new CropStream(s2, TimeUtil.TimeSpanToBytes(t2From, s2.Properties), TimeUtil.TimeSpanToBytes(t2To, s2.Properties)); List <Tuple <TimeSpan, TimeSpan> > path = null; DTW dtw = null; // execute time warping if (type == TimeWarpType.DTW) { dtw = new DTW(TimeWarpSearchWidth, progressMonitor); } else if (type == TimeWarpType.OLTW) { dtw = new OLTW2(TimeWarpSearchWidth, progressMonitor); } if (TimeWarpDisplay) { this.Dispatcher.BeginInvoke((Action) delegate { dtwPathViewer = new DtwPathViewer(); dtwPathViewer.Show(); }); dtw.OltwInit += new DTW.OltwInitDelegate(delegate(int windowSize, IMatrix <double> cellCostMatrix, IMatrix <double> totalCostMatrix) { dtwPathViewer.Dispatcher.BeginInvoke((Action) delegate { dtwPathViewer.DtwPath.Init(windowSize, cellCostMatrix, totalCostMatrix); }); }); bool drawing = false; dtw.OltwProgress += new DTW.OltwProgressDelegate(delegate(int i, int j, int minI, int minJ, bool force) { if (!drawing || force) { dtwPathViewer.Dispatcher.BeginInvoke((Action) delegate { drawing = true; dtwPathViewer.DtwPath.Refresh(i, j, minI, minJ); drawing = false; }); } }); } path = dtw.Execute(s1, s2); if (path == null) { return; } // convert resulting path to matches and filter them int filterSize = TimeWarpFilterSize; // take every n-th match and drop the rest bool smoothing = TimeWarpSmoothing; int smoothingWidth = Math.Max(1, Math.Min(filterSize / 10, filterSize)); bool inOutCue = TimeWarpInOutCue; TimeSpan inOutCueSpan = TimeWarpSearchWidth; List <Match> matches = new List <Match>(); float maxSimilarity = 0; // needed for normalization IProgressReporter progressReporter = progressMonitor.BeginTask("post-process resulting path...", true); double totalProgress = path.Count; double progress = 0; /* Leave out matches in the in/out cue areas... * The matches in the interval at the beginning and end of the calculated time warping path with a width * equal to the search width should be left out because they might not be correct - since the time warp * path needs to start at (0,0) in the matrix and end at (m,n), they would only be correct if the path gets * calculated between two synchronization points. Paths calculated from the start of a track to the first * sync point, or from the last sync point to end of the track are probably wrong in this interval since * the start and end points don't match if there is time drift so it is better to leave them out in those * areas... in those short a few second long intervals the drict actually will never be that extreme that * someone would notice it anyway. */ if (inOutCue) { int startIndex = 0; int endIndex = path.Count; // this needs a temporally ordered mapping path (no matter if ascending or descending) foreach (Tuple <TimeSpan, TimeSpan> mapping in path) { if (cueIn && (mapping.Item1 < inOutCueSpan || mapping.Item2 < inOutCueSpan)) { startIndex++; } if (cueOut && (mapping.Item1 > (t1To - t1From - inOutCueSpan) || mapping.Item2 > (t2To - t2From - inOutCueSpan))) { endIndex--; } } path = path.GetRange(startIndex, endIndex - startIndex); } for (int i = 0; i < path.Count; i += filterSize) { //List<Tuple<TimeSpan, TimeSpan>> section = path.GetRange(i, Math.Min(path.Count - i, filterSize)); List <Tuple <TimeSpan, TimeSpan> > smoothingSection = path.GetRange( Math.Max(0, i - smoothingWidth / 2), Math.Min(path.Count - i, smoothingWidth)); Tuple <TimeSpan, TimeSpan> match = path[i]; if (smoothingSection.Count == 0) { throw new InvalidOperationException("must not happen"); } else if (smoothingSection.Count == 1 || !smoothing || i == 0) { // do nothing, match doesn't need any processing // the first and last match must not be smoothed since they must sit at the bounds } else { List <TimeSpan> offsets = new List <TimeSpan>(smoothingSection.Select(t => t.Item2 - t.Item1).OrderBy(t => t)); int middle = offsets.Count / 2; // calculate median // http://en.wikiversity.org/wiki/Primary_mathematics/Average,_median,_and_mode#Median TimeSpan smoothedDriftTime = new TimeSpan((offsets[middle - 1] + offsets[middle]).Ticks / 2); match = new Tuple <TimeSpan, TimeSpan>(match.Item1, match.Item1 + smoothedDriftTime); } float similarity = calculateSimilarity ? (float)Math.Abs(CrossCorrelation.Correlate( s1, new Interval(match.Item1.Ticks, match.Item1.Ticks + TimeUtil.SECS_TO_TICKS), s2, new Interval(match.Item2.Ticks, match.Item2.Ticks + TimeUtil.SECS_TO_TICKS))) : 1; if (similarity > maxSimilarity) { maxSimilarity = similarity; } matches.Add(new Match() { Track1 = t1, Track1Time = match.Item1 + t1From, Track2 = t2, Track2Time = match.Item2 + t2From, Similarity = similarity, Source = type.ToString() }); progressReporter.ReportProgress(progress / totalProgress * 100); progress += filterSize; } // add last match if it hasn't been added if (path.Count > 0 && path.Count % filterSize != 1) { Tuple <TimeSpan, TimeSpan> lastMatch = path[path.Count - 1]; matches.Add(new Match() { Track1 = t1, Track1Time = lastMatch.Item1 + t1From, Track2 = t2, Track2Time = lastMatch.Item2 + t2From, Similarity = 1, Source = type.ToString() }); } progressReporter.Finish(); multiTrackViewer.Dispatcher.BeginInvoke((Action) delegate { foreach (Match match in matches) { if (normalizeSimilarity) { match.Similarity /= maxSimilarity; // normalize to 1 } multiTrackViewer.Matches.Add(match); } }); s1.Close(); s2.Close(); }
private void dtwButton_Click(object sender, RoutedEventArgs e) { if (trackList.Count > 1) { TimeWarpType type = TimeWarpType.DTW; if ((bool)dtwRadioButton.IsChecked) { type = TimeWarpType.DTW; } else if ((bool)oltwRadioButton.IsChecked) { type = TimeWarpType.OLTW; } TimeWarpMode mode = TimeWarpMode.SelectedTracks; if ((bool)timeWarpModeBorderSectionsRadioButton.IsChecked) { mode = TimeWarpMode.BorderSections; } else if ((bool)timeWarpModeAllSectionsRadioButton.IsChecked) { mode = TimeWarpMode.AllSections; } bool calculateSimilarity = (bool)dtwSimilarityCheckBox.IsChecked; bool normalizeSimilarity = (bool)dtwSimilarityNormalizationCheckBox.IsChecked; if (mode == TimeWarpMode.SelectedTracks) { if (trackList.Count > 1) { var masterTrack = (AudioTrack)multiTrackViewer.SelectedItem; var slaveTracks = new List <AudioTrack>(multiTrackViewer.SelectedItems.Cast <AudioTrack>()); slaveTracks.Remove(masterTrack); Task.Factory.StartNew(() => { Parallel.ForEach(slaveTracks, new ParallelOptions { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) }, slaveTrack => { TimeWarp(type, masterTrack, TimeSpan.Zero, masterTrack.Length, slaveTrack, TimeSpan.Zero, slaveTrack.Length, calculateSimilarity, normalizeSimilarity, false, false); }); }); } } else { List <MatchGroup> trackGroups = DetermineMatchGroups(MatchFilterMode.None); Task.Factory.StartNew(() => { foreach (MatchGroup trackGroup in trackGroups) { foreach (MatchPair trackPair in trackGroup.MatchPairs) { List <Match> matches = trackPair.Matches.OrderBy(match => match.Track1Time).ToList(); Match first = matches.First(); Match last = matches.Last(); //Task.Factory.StartNew(() => { TimeSpan sectionLength = first.Track1Time > first.Track2Time ? first.Track2Time : first.Track1Time; TimeWarp(type, first.Track1, first.Track1Time - sectionLength, first.Track1Time, first.Track2, first.Track2Time - sectionLength, first.Track2Time, calculateSimilarity, normalizeSimilarity, true, false); //}); if (mode == TimeWarpMode.AllSections) { if (matches.Count > 1) { for (int i = 0; i < matches.Count - 1; i++) { Match from = matches[i]; Match to = matches[i + 1]; TimeWarp(type, from.Track1, from.Track1Time, to.Track1Time, from.Track2, from.Track2Time, to.Track2Time, calculateSimilarity, normalizeSimilarity, false, false); } } } //Task.Factory.StartNew(() => { sectionLength = last.Track1.Length - last.Track1Time > last.Track2.Length - last.Track2Time ? last.Track2.Length - last.Track2Time : last.Track1.Length - last.Track1Time; TimeWarp(type, last.Track1, last.Track1Time, last.Track1Time + sectionLength, last.Track2, last.Track2Time, last.Track2Time + sectionLength, calculateSimilarity, normalizeSimilarity, false, true); //}); } } }); } } }