public override List <Tuple <TimeSpan, TimeSpan> > Execute(IAudioStream s1, IAudioStream s2) { s1 = PrepareStream(s1); s2 = PrepareStream(s2); int searchWidth = (int)(this.searchWidth.TotalSeconds * (1d * FrameReader.SAMPLERATE / FrameReader.WINDOW_HOP_SIZE)); totalCostMatrix = new PatchMatrix <double>(double.PositiveInfinity); cellCostMatrix = new PatchMatrix <double>(double.PositiveInfinity); rb1 = new RingBuffer <float[]>(searchWidth); rb1FrameCount = 0; rb2 = new RingBuffer <float[]>(searchWidth); rb2FrameCount = 0; stream1FrameQueue = new BlockingCollection <float[]>(20); FrameReader stream1FrameReader = new FrameReader(s1); Task.Factory.StartNew(() => { while (stream1FrameReader.HasNext()) { float[] frame = new float[FrameReader.FRAME_SIZE]; stream1FrameReader.ReadFrame(frame); Thread.Sleep(20); stream1FrameQueue.Add(frame); } stream1FrameQueue.CompleteAdding(); }); stream2FrameQueue = new BlockingCollection <float[]>(20); FrameReader stream2FrameReader = new FrameReader(s2); Task.Factory.StartNew(() => { while (stream2FrameReader.HasNext()) { float[] frame = new float[FrameReader.FRAME_SIZE]; stream2FrameReader.ReadFrame(frame); Thread.Sleep(20); stream2FrameQueue.Add(frame); } stream2FrameQueue.CompleteAdding(); }); // init matrix // NOTE do not explicitely init the PatchMatrix, otherwise the sparse matrix characteristic would // be gone and the matrix would take up all the space like a standard matrix does totalCostMatrix[0, 0] = 0; cellCostMatrix[0, 0] = 0; IProgressReporter progressReporter = progressMonitor.BeginTask("OLTW", true); int totalFrames = stream1FrameReader.WindowCount + stream2FrameReader.WindowCount; // --------- OLTW ----------- t = 1; j = 1; previous = GetIncResult.None; runCount = 0; c = searchWidth; EvaluatePathCost(t, j); FireOltwInit(c, cellCostMatrix, totalCostMatrix); while (rb1FrameCount + rb2FrameCount < totalFrames) { GetIncResult getInc = GetInc(t, j); if (t < stream1FrameReader.WindowCount && getInc != GetIncResult.Column) { t++; for (int k = j - c + 1; k <= j; k++) { if (k > 0) { EvaluatePathCost(t, k); } } } if (j < stream2FrameReader.WindowCount && getInc != GetIncResult.Row) { j++; for (int k = t - c + 1; k <= t; k++) { if (k > 0) { EvaluatePathCost(k, j); } } } if (getInc == previous) { runCount++; } else { runCount = 1; } if (getInc != GetIncResult.Both) { previous = getInc; } FireOltwProgress(t, j, t, j, false); //Debug.WriteLine(t + " " + j + " " + getInc); progressReporter.ReportProgress((rb1FrameCount + rb2FrameCount) / (double)totalFrames * 100); } FireOltwProgress(t, j, t, j, true); Debug.WriteLine("OLTW finished @ t={0}/{1}, j={2}/{3}", t, stream1FrameReader.WindowCount, j, stream2FrameReader.WindowCount); progressReporter.Finish(); // --------- generate results ----------- List <Pair> path = OptimalWarpingPath(totalCostMatrix); path.Reverse(); List <Tuple <TimeSpan, TimeSpan> > pathTimes = new List <Tuple <TimeSpan, TimeSpan> >(); foreach (Pair pair in path) { Tuple <TimeSpan, TimeSpan> timePair = new Tuple <TimeSpan, TimeSpan>( PositionToTimeSpan(pair.i1 * FrameReader.WINDOW_HOP_SIZE), PositionToTimeSpan(pair.i2 * FrameReader.WINDOW_HOP_SIZE)); if (timePair.Item1 >= TimeSpan.Zero && timePair.Item2 >= TimeSpan.Zero) { pathTimes.Add(timePair); } } return(pathTimes); }
public virtual List<Tuple<TimeSpan, TimeSpan>> Execute(IAudioStream s1, IAudioStream s2) { s1 = PrepareStream(s1); s2 = PrepareStream(s2); int diagonalWidth = (int)(searchWidth.TotalSeconds * (1d * FrameReader.SAMPLERATE / FrameReader.WINDOW_HOP_SIZE)); // init ring buffers rb1 = new float[diagonalWidth][]; rb1FrameCount = 0; rb2 = new float[diagonalWidth][]; rb2FrameCount = 0; stream1FrameQueue = new BlockingCollection<float[]>(20); FrameReader stream1FrameReader = new FrameReader(s1); Task.Factory.StartNew(() => { while (stream1FrameReader.HasNext()) { float[] frame = new float[FrameReader.FRAME_SIZE]; stream1FrameReader.ReadFrame(frame); //Thread.Sleep(20); stream1FrameQueue.Add(frame); } stream1FrameQueue.CompleteAdding(); }); stream2FrameQueue = new BlockingCollection<float[]>(20); FrameReader stream2FrameReader = new FrameReader(s2); Task.Factory.StartNew(() => { while (stream2FrameReader.HasNext()) { float[] frame = new float[FrameReader.FRAME_SIZE]; stream2FrameReader.ReadFrame(frame); //Thread.Sleep(20); stream2FrameQueue.Add(frame); } stream2FrameQueue.CompleteAdding(); }); IProgressReporter progressReporter = progressMonitor.BeginTask("DTW aligning...", true); int n = stream1FrameReader.WindowCount; int m = stream2FrameReader.WindowCount; double deltaN; double deltaM; if (m > n) { deltaN = (double)(n - 1) / (m - 1); deltaM = 1d; } else if (m < n) { deltaN = 1d; deltaM = (double)(m - 1) / (n - 1); } else { deltaN = 1d; deltaM = 1d; } // NOTE the SparseMatrix is NOT USABLE for DTW -> OutOfMemoryException (1.3GB RAM) at a densely filled matrix of ~4000x3000 IMatrix<double> totalCostMatrix = new PatchMatrix<double>(double.PositiveInfinity); IMatrix<double> cellCostMatrix = new PatchMatrix<double>(double.PositiveInfinity); // init matrix // NOTE do not explicitely init the PatchMatrix, otherwise the sparse matrix characteristic would // be gone and the matrix would take up all the space like a standard matrix does totalCostMatrix[0, 0] = 0; cellCostMatrix[0, 0] = 0; double progressN = 0; double progressM = 0; int i, x = 0; int j, y = 0; FireOltwInit(diagonalWidth, cellCostMatrix, totalCostMatrix); while (x < n || y < m) { x = (int)progressN + 1; y = (int)progressM + 1; ReadFrames(x, y); i = Math.Max(x - diagonalWidth, 1); j = y; for (; i <= x; i++) { double cost = CalculateCost(rb1[(i - 1) % diagonalWidth], rb2[(j - 1) % diagonalWidth]); totalCostMatrix[i, j] = cost + Min( totalCostMatrix[i - 1, j], totalCostMatrix[i, j - 1], totalCostMatrix[i - 1, j - 1]); cellCostMatrix[i, j] = cost; } i = x; j = Math.Max(y - diagonalWidth, 1); for (; j <= y; j++) { double cost = CalculateCost(rb1[(i - 1) % diagonalWidth], rb2[(j - 1) % diagonalWidth]); totalCostMatrix[i, j] = cost + Min( totalCostMatrix[i - 1, j], totalCostMatrix[i, j - 1], totalCostMatrix[i - 1, j - 1]); cellCostMatrix[i, j] = cost; } FireOltwProgress(x, y, x, y, false); progressReporter.ReportProgress((double)x / n * 100); progressN += deltaN; progressM += deltaM; } FireOltwProgress(x, y, x, y, true); progressReporter.Finish(); List<Pair> path = OptimalWarpingPath(totalCostMatrix); path.Reverse(); return WarpingPathTimes(path, true); }
public override List <Tuple <TimeSpan, TimeSpan> > Execute(IAudioStream s1, IAudioStream s2) { s1 = PrepareStream(s1); s2 = PrepareStream(s2); IProgressReporter progressReporter; progressReporter = progressMonitor.BeginTask("OLTW analyzing stream 1..", true); FrameReader s1FrameReader = new FrameReader(s1); s1Frames = new List <float[]>(s1FrameReader.WindowCount); int s1FrameCount = 0; while (s1FrameReader.HasNext()) { float[] frame = new float[FrameReader.FRAME_SIZE]; s1FrameReader.ReadFrame(frame); s1Frames.Add(frame); progressReporter.ReportProgress((double)++s1FrameCount / s1Frames.Capacity * 100); } progressReporter.Finish(); progressReporter = progressMonitor.BeginTask("OLTW analyzing stream 2...", true); FrameReader s2FrameReader = new FrameReader(s2); s2Frames = new List <float[]>(s2FrameReader.WindowCount); int s2FrameCount = 0; while (s2FrameReader.HasNext()) { float[] frame = new float[FrameReader.FRAME_SIZE]; s2FrameReader.ReadFrame(frame); s2Frames.Add(frame); progressReporter.ReportProgress((double)++s2FrameCount / s2Frames.Capacity * 100); } progressReporter.Finish(); int c = (int)(this.searchWidth.TotalSeconds * (1d * FrameReader.SAMPLERATE / FrameReader.WINDOW_HOP_SIZE)); c = Math.Min(c, Math.Min(s1FrameCount, s2FrameCount)); // reduce c to the shortest stream if necessary totalCostMatrix = new PatchMatrix <double>(double.PositiveInfinity, 100); cellCostMatrix = new PatchMatrix <double>(double.PositiveInfinity, 100); int i = 0, j = 0; // position of the "head" in each step int minI, minJ; // position of the cell with the min path cost in each step Direction r = Direction.None; Direction previous = Direction.None; int runCount = 0; pathLengthRow = new int[c + MAX_RUN_COUNT]; pathLengthCol = new int[c + MAX_RUN_COUNT]; int[] pathLengthRowPrev = new int[c + MAX_RUN_COUNT]; int[] pathLengthColPrev = new int[c + MAX_RUN_COUNT]; int totalFrames = s1Frames.Count + s2Frames.Count; progressReporter = progressMonitor.BeginTask("OLTW aligning...", true); // init totalCostMatrix[0, 0] = cellCostMatrix[0, 0] = CalculateCost(s1Frames[0], s2Frames[0]); FireOltwInit(c, cellCostMatrix, totalCostMatrix); while (i < s1Frames.Count - 1 || j < s2Frames.Count - 1) { if (i < c) // build initial square matrix { r = Direction.Both; minI = i; minJ = j; } else // calculate temp. min cost path { int xI = i; double xV = double.PositiveInfinity; for (int x = i - c + 1; x <= i; x++) { double pC = totalCostMatrix[x, j] / pathLengthRow[x % pathLengthRow.Length]; if (pC <= xV) { xV = pC; xI = x; } } int yI = j; double yV = double.PositiveInfinity; for (int y = j - c + 1; y <= j; y++) { double pC = totalCostMatrix[i, y] / pathLengthCol[y % pathLengthRow.Length]; if (pC <= yV) { yV = pC; yI = y; } } if (xI == i && yI == j) { // min path is in the corner r = Direction.Both; minI = i; minJ = j; } else if (xV < yV) { // min path was found in a row r = Direction.Row; minI = xI; minJ = j; } else { // min path is in the column r = Direction.Column; minI = i; minJ = yI; } } FireOltwProgress(i, j, minI, minJ, false); if (r == previous) { runCount++; } if (r == Direction.Both) { runCount = 0; } else if (runCount >= MAX_RUN_COUNT) { if (r == Direction.Row) { r = Direction.Column; } else if (r == Direction.Column) { r = Direction.Row; } runCount = 0; } // add row if (j < s2Frames.Count - 1 && (r == Direction.Row || r == Direction.Both)) { CommonUtil.Swap <int[]>(ref pathLengthRow, ref pathLengthRowPrev); j++; for (int x = Math.Max(i - c + 1, 0); x <= i; x++) { double cellCost = CalculateCost(s1Frames[x], s2Frames[j]); if (x == 0) { totalCostMatrix[x, j] = totalCostMatrix[x, j - 1] + cellCost; } else { totalCostMatrix[x, j] = Min( totalCostMatrix[x - 1, j - 1] + DIAG_COST_FACTOR * cellCost, totalCostMatrix[x - 1, j] + cellCost, totalCostMatrix[x, j - 1] + cellCost); } cellCostMatrix[x, j] = cellCost; pathLengthRow[x % pathLengthRow.Length] = GetPathLength(x, j); } pathLengthCol[j % pathLengthCol.Length] = pathLengthRow[i % pathLengthRow.Length]; } // add column if (i < s1Frames.Count - 1 && (r == Direction.Column || r == Direction.Both)) { CommonUtil.Swap <int[]>(ref pathLengthCol, ref pathLengthColPrev); i++; for (int y = Math.Max(j - c + 1, 0); y <= j; y++) { double cellCost = CalculateCost(s1Frames[i], s2Frames[y]); if (y == 0) { totalCostMatrix[i, y] = totalCostMatrix[i - 1, y] + cellCost; } else { totalCostMatrix[i, y] = Min( totalCostMatrix[i - 1, y - 1] + DIAG_COST_FACTOR * cellCost, totalCostMatrix[i - 1, y] + cellCost, totalCostMatrix[i, y - 1] + cellCost); } cellCostMatrix[i, y] = cellCost; pathLengthCol[y % pathLengthCol.Length] = GetPathLength(i, y); } pathLengthRow[i % pathLengthRow.Length] = pathLengthCol[j % pathLengthCol.Length]; } previous = r; progressReporter.ReportProgress((i + j) / (double)totalFrames * 100); //Thread.Sleep(2); } FireOltwProgress(i, j, i, j, true); progressReporter.Finish(); List <Pair> path = OptimalWarpingPath(totalCostMatrix); path.Reverse(); return(WarpingPathTimes(path, true)); }