Exemple #1
0
        private TensorFloat TensorizeWithVideoFrame()
        {
            var s = new System.Diagnostics.Stopwatch();

            s.Start();
            var bitmap          = decoder_.GetSoftwareBitmapAsync(BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied).GetAwaiter().GetResult();
            var videoFrame      = VideoFrame.CreateWithSoftwareBitmap(bitmap);
            var outputTensorI64 = TensorInt64Bit.Create();

            Evaluate(shapeSession_, videoFrame, outputTensorI64);
            s.Stop();
            PerformanceMetricsMonitor.Log("VideoFrame Tensorization", s.ElapsedMilliseconds * 1000000);
            return(null);
        }
Exemple #2
0
        static SoftwareBitmap GetMelspectrogramFromSignal(
            IEnumerable <float> rawSignal,
            int batchSize    = 1,
            int windowSize   = 256,
            int dftSize      = 256,
            int hopSize      = 3,
            int nMelBins     = 1024,
            int samplingRate = 8192,
            int amplitude    = 5000
            )
        {
            float[] signal = rawSignal.ToArray();

            //Scale the signal by a given amplitude
            for (int i = 0; i < signal.Length; i++)
            {
                signal[i] = signal[i] * amplitude;
            }

            int signalSize      = signal.Length;
            var nDFT            = 1 + (signalSize - dftSize) / hopSize;
            var onesidedDftSize = (dftSize >> 1) + 1;

            long[] signalShape         = { batchSize, signalSize };
            long[] melSpectrogramShape = { batchSize, 1, nDFT, nMelBins };

            var builder = LearningModelBuilder.Create(13)
                          .Inputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Input.TimeSignal", TensorKind.Float, signalShape))
                          .Outputs.Add(LearningModelBuilder.CreateTensorFeatureDescriptor("Output.MelSpectrogram", TensorKind.Float, melSpectrogramShape))
                          .Operators.Add(new Operator("HannWindow", MicrosoftExperimentalDomain)
                                         .SetConstant("size", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { windowSize }))
                                         .SetOutput("output", "hann_window"))
                          .Operators.Add(new Operator("STFT", MicrosoftExperimentalDomain)
                                         .SetName("STFT_NAMED_NODE")
                                         .SetInput("signal", "Input.TimeSignal")
                                         .SetInput("window", "hann_window")
                                         .SetConstant("frame_length", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { dftSize }))
                                         .SetConstant("frame_step", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { hopSize }))
                                         .SetOutput("output", "stft_output"))
                          .Operators.Add(new Operator("ReduceSumSquare")
                                         .SetInput("data", "stft_output")
                                         .SetAttribute("axes", TensorInt64Bit.CreateFromArray(new List <long>()
            {
                1
            }, new long[] { 3 }))
                                         .SetAttribute("keepdims", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { 0 }))
                                         .SetOutput("reduced", "magnitude_squared"))
                          .Operators.Add(new Operator("Div")
                                         .SetInput("A", "magnitude_squared")
                                         .SetConstant("B", TensorFloat.CreateFromArray(new List <long>(), new float[] { dftSize }))
                                         .SetOutput("C", "power_frames"))
                          .Operators.Add(new Operator("MelWeightMatrix", MicrosoftExperimentalDomain)
                                         .SetConstant("num_mel_bins", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { nMelBins }))
                                         .SetConstant("dft_length", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { dftSize }))
                                         .SetConstant("sample_rate", TensorInt64Bit.CreateFromArray(new List <long>(), new long[] { samplingRate }))
                                         .SetConstant("lower_edge_hertz", TensorFloat.CreateFromArray(new List <long>(), new float[] { 0 }))
                                         .SetConstant("upper_edge_hertz", TensorFloat.CreateFromArray(new List <long>(), new float[] { (float)(samplingRate / 2.0) }))
                                         .SetOutput("output", "mel_weight_matrix"))
                          .Operators.Add(new Operator("Reshape")
                                         .SetInput("data", "power_frames")
                                         .SetConstant("shape", TensorInt64Bit.CreateFromArray(new List <long>()
            {
                2
            }, new long[] { batchSize *nDFT, onesidedDftSize }))
                                         .SetOutput("reshaped", "reshaped_output"))
                          .Operators.Add(new Operator("MatMul")
                                         .SetInput("A", "reshaped_output")
                                         .SetInput("B", "mel_weight_matrix")
                                         .SetOutput("Y", "mel_spectrogram"))
                          .Operators.Add(new Operator("Reshape")
                                         .SetInput("data", "mel_spectrogram")
                                         .SetConstant("shape", TensorInt64Bit.CreateFromArray(new List <long>()
            {
                4
            }, melSpectrogramShape))
                                         .SetOutput("reshaped", "Output.MelSpectrogram"));

            var model = builder.CreateModel();

            LearningModelSession session = new LearningModelSession(model);
            LearningModelBinding binding = new LearningModelBinding(session);

            // Bind input
            binding.Bind("Input.TimeSignal", TensorFloat.CreateFromArray(signalShape, signal));

            // Bind output
            var outputImage = new VideoFrame(
                BitmapPixelFormat.Bgra8,
                nMelBins,
                nDFT);

            binding.Bind("Output.MelSpectrogram", outputImage);

            // Evaluate
            var sw     = Stopwatch.StartNew();
            var result = session.Evaluate(binding, "");

            sw.Stop();
            Console.WriteLine("Evaluate Took: %f\n", sw.ElapsedMilliseconds);

            return(outputImage.SoftwareBitmap);
        }