Ejemplo n.º 1
0
        public static void GenerateSerialCDExample(string simulatedLocation, string outputFolder)
        {
            var data     = MeasurementData.LoadSimulatedPoints(simulatedLocation);
            var cellSize = 1.0 / 3600.0 * Math.PI / 180.0;
            var c        = new GriddingConstants(data.VisibilitiesCount, 256, 8, 4, 512, (float)cellSize, 1, 0.0);
            var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);

            var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies);
            var psf     = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);
            var corrKernel = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, c.GridSize, c.GridSize));

            Directory.CreateDirectory(outputFolder);
            var reconstruction = new float[c.GridSize, c.GridSize];
            var residualVis    = data.Visibilities;
            var totalSize      = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var fastCD         = new FastSerialCD(totalSize, psf);
            var lambda         = 0.50f * fastCD.MaxLipschitz;
            var alpha          = 0.2f;

            for (int cycle = 0; cycle < 100; cycle++)
            {
                var dirtyGrid  = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies);
                var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                var gradients = Residuals.CalcGradientMap(dirtyImage, corrKernel, totalSize);

                Tools.WriteToMeltCSV(Common.PSF.Cut(reconstruction), Path.Combine(outputFolder, "model_CD_" + cycle + ".csv"));
                Tools.WriteToMeltCSV(gradients, Path.Combine(outputFolder, "gradients_CD_" + cycle + ".csv"));

                fastCD.Deconvolve(reconstruction, gradients, lambda, alpha, 4);

                FFT.Shift(reconstruction);
                var xGrid = FFT.Forward(reconstruction);
                FFT.Shift(reconstruction);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies);
                residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
            }
        }
        private static void ReconstructMinorCycle(MeasurementData input, GriddingConstants c, int cutFactor, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, bool useAccelerated = true, int blockSize = 1, int maxCycle = 6)
        {
            var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);

            var totalSize    = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var psfCut       = PSF.Cut(fullPsf, cutFactor);
            var maxSidelobe  = PSF.CalcMaxSidelobe(fullPsf, cutFactor);
            var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2);
            var random       = new Random(123);
            var approx       = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.1f, searchPercent, false, useAccelerated);

            using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
                using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
                    using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf))
                    {
                        var currentBMapCalculator = bMapCalculator;

                        var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
                        var lambda       = (float)(LAMBDA * maxLipschitz);
                        var lambdaTrue   = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
                        var alpha        = ALPHA;
                        ApproxFast.LAMBDA_TEST = lambdaTrue;
                        ApproxFast.ALPHA_TEST  = alpha;

                        var switchedToOtherPsf = false;
                        var writer             = new StreamWriter(folder + "/" + file + "_lambda.txt");
                        var data        = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt"));
                        var xImage      = new float[c.GridSize, c.GridSize];
                        var residualVis = input.Visibilities;
                        for (int cycle = 0; cycle < maxCycle; cycle++)
                        {
                            Console.WriteLine("cycle " + cycle);
                            var dirtyGrid  = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
                            var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
                            FFT.Shift(dirtyImage);
                            FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");

                            var minLambda     = 0.0f;
                            var dirtyCopy     = Copy(dirtyImage);
                            var xCopy         = Copy(xImage);
                            var currentLambda = 0f;
                            //var residualsConvolver = new PaddedConvolver(PSF.CalcPaddedFourierConvolution(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)));
                            for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++)
                            {
                                FitsIO.Write(dirtyImage, folder + "/dirtyMinor_" + minorCycle + ".fits");
                                var maxDirty         = Residuals.GetMax(dirtyImage);
                                var bMap             = currentBMapCalculator.Convolve(dirtyImage);
                                var maxB             = Residuals.GetMax(bMap);
                                var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
                                var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                                currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);

                                if (minorCycle == 0)
                                {
                                    minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
                                }

                                if (currentLambda < minLambda)
                                {
                                    currentLambda = minLambda;
                                }

                                writer.WriteLine(cycle + ";" + minorCycle + ";" + currentLambda + ";" + minLambda);
                                writer.Flush();
                                approx.DeconvolveTest(data, cycle, minorCycle, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f);
                                FitsIO.Write(xImage, folder + "/xImageMinor_" + minorCycle + ".fits");

                                if (currentLambda == lambda | currentLambda == minLambda)
                                {
                                    break;
                                }

                                Console.WriteLine("resetting residuals!!");
                                //reset dirtyImage with full PSF
                                var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
                                Parallel.For(0, xCopy.GetLength(0), (i) =>
                                {
                                    for (int j = 0; j < xCopy.GetLength(1); j++)
                                    {
                                        residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
                                    }
                                });
                                residualsConvolver.ConvolveInPlace(residualsUpdate);

                                Parallel.For(0, xCopy.GetLength(0), (i) =>
                                {
                                    for (int j = 0; j < xCopy.GetLength(1); j++)
                                    {
                                        dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
                                    }
                                });
                            }

                            if (currentLambda == lambda & !switchedToOtherPsf)
                            {
                                approx.ResetAMap(fullPsf);
                                currentBMapCalculator = bMapCalculator2;
                                lambda             = lambdaTrue;
                                switchedToOtherPsf = true;
                                writer.WriteLine("switched");
                                writer.Flush();
                            }

                            FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits");

                            FFT.Shift(xImage);
                            var xGrid = FFT.Forward(xImage);
                            FFT.Shift(xImage);
                            var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
                            residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
                        }

                        writer.Close();
                    }
        }
        private static void Reconstruct(Data input, int cutFactor, float[,] fullPsf, string folder, string file, int threads, int blockSize, bool accelerated, float randomPercent, float searchPercent)
        {
            var totalSize      = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize);
            var psfCut         = PSF.Cut(fullPsf, cutFactor);
            var maxSidelobe    = PSF.CalcMaxSidelobe(fullPsf, cutFactor);
            var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
            var random         = new Random(123);
            var approx         = new ApproxFast(totalSize, psfCut, threads, blockSize, randomPercent, searchPercent, false, true);

            var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut);
            var lambda          = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut));
            var lambdaTrue      = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
            var alpha           = ALPHA;

            ApproxFast.LAMBDA_TEST = lambdaTrue;
            ApproxFast.ALPHA_TEST  = alpha;

            var switchedToOtherPsf = false;
            var writer             = new StreamWriter(folder + "/" + file + "_lambda.txt");
            var data        = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt"));
            var xImage      = new float[input.c.GridSize, input.c.GridSize];
            var residualVis = input.visibilities;

            for (int cycle = 0; cycle < 7; cycle++)
            {
                Console.WriteLine("cycle " + cycle);
                var dirtyGrid  = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies);
                var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");

                var maxDirty         = Residuals.GetMax(dirtyImage);
                var bMap             = bMapCalculator.Convolve(dirtyImage);
                var maxB             = Residuals.GetMax(bMap);
                var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f);
                var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                var currentLambda    = (float)Math.Max(currentSideLobe / alpha, lambda);

                writer.WriteLine("cycle" + ";" + currentLambda);
                writer.Flush();

                approx.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f);
                FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits");

                if (currentLambda == lambda & !switchedToOtherPsf)
                {
                    approx.ResetAMap(fullPsf);
                    lambda             = lambdaTrue;
                    switchedToOtherPsf = true;
                    writer.WriteLine("switched");
                    writer.Flush();
                }

                FFT.Shift(xImage);
                var xGrid = FFT.Forward(xImage);
                FFT.Shift(xImage);
                var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies);
                residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags);
            }
            writer.Close();
        }
Ejemplo n.º 4
0
        public static void GeneratePSFs(string simulatedLocation, string outputFolder)
        {
            var data     = MeasurementData.LoadSimulatedPoints(simulatedLocation);
            var c        = MeasurementData.CreateSimulatedStandardParams(data.VisibilitiesCount);
            var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);

            var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies);
            var psf     = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);

            Directory.CreateDirectory(outputFolder);

            var maskedPsf = Copy(psf);

            Tools.Mask(maskedPsf, 2);
            var reverseMasked = Copy(psf);

            Tools.ReverseMask(reverseMasked, 2);
            var psf2    = PSF.CalcPSFSquared(psf);
            var psf2Cut = PSF.CalcPSFSquared(maskedPsf);

            Tools.WriteToMeltCSV(psf, Path.Combine(outputFolder, "psf.csv"));
            Tools.WriteToMeltCSV(maskedPsf, Path.Combine(outputFolder, "psfCut.csv"));
            Tools.WriteToMeltCSV(reverseMasked, Path.Combine(outputFolder, "psfReverseCut.csv"));
            Tools.WriteToMeltCSV(psf2, Path.Combine(outputFolder, "psfSquared.csv"));
            Tools.WriteToMeltCSV(psf2Cut, Path.Combine(outputFolder, "psfSquaredCut.csv"));

            var x = new float[c.GridSize, c.GridSize];

            x[10, 10] = 1.0f;

            var convKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, c.GridSize, c.GridSize));
            var corrKernel = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, c.GridSize, c.GridSize));

            using (var convolver = new PaddedConvolver(convKernel, new Rectangle(0, 0, c.GridSize, c.GridSize)))
                using (var correlator = new PaddedConvolver(corrKernel, new Rectangle(0, 0, c.GridSize, c.GridSize)))
                {
                    var zeroPadded = convolver.Convolve(x);
                    var psf2Edge   = correlator.Convolve(zeroPadded);
                    Tools.WriteToMeltCSV(zeroPadded, Path.Combine(outputFolder, "psfZeroPadding.csv"));
                    Tools.WriteToMeltCSV(psf2Edge, Path.Combine(outputFolder, "psfSquaredEdge.csv"));
                }
            convKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, 0, 0));
            using (var convolver = new PaddedConvolver(convKernel, new Rectangle(0, 0, 0, 0)))
                Tools.WriteToMeltCSV(convolver.Convolve(x), Path.Combine(outputFolder, "psfCircular.csv"));

            //================================================= Reconstruct =============================================================
            var totalSize      = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var reconstruction = new float[c.GridSize, c.GridSize];
            var fastCD         = new FastSerialCD(totalSize, psf);
            var lambda         = 0.50f * fastCD.MaxLipschitz;
            var alpha          = 0.2f;

            var residualVis = data.Visibilities;

            for (int cycle = 0; cycle < 5; cycle++)
            {
                Console.WriteLine("in cycle " + cycle);
                var dirtyGrid  = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies);
                var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
                FFT.Shift(dirtyImage);

                var gradients = Residuals.CalcGradientMap(dirtyImage, corrKernel, totalSize);

                if (cycle == 0)
                {
                    Tools.WriteToMeltCSV(dirtyImage, Path.Combine(outputFolder, "dirty.csv"));
                    Tools.WriteToMeltCSV(gradients, Path.Combine(outputFolder, "gradients.csv"));
                }

                fastCD.Deconvolve(reconstruction, gradients, lambda, alpha, 10000, 1e-5f);

                FFT.Shift(reconstruction);
                var xGrid = FFT.Forward(reconstruction);
                FFT.Shift(reconstruction);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies);
                residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
            }

            //FitsIO.Write(reconstruction, Path.Combine(outputFolder,"xImage.fits"));
            Tools.WriteToMeltCSV(reconstruction, Path.Combine(outputFolder, "elasticNet.csv"));
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Major cycle implementation for the Serial CD
        /// </summary>
        /// <param name="obsName"></param>
        /// <param name="data"></param>
        /// <param name="c"></param>
        /// <param name="useGPU"></param>
        /// <param name="psfCutFactor"></param>
        /// <param name="maxMajorCycle"></param>
        /// <param name="lambda"></param>
        /// <param name="alpha"></param>
        /// <param name="deconvIterations"></param>
        /// <param name="deconvEpsilon"></param>
        public static void ReconstructSerialCD(string obsName, MeasurementData data, GriddingConstants c, bool useGPU, int psfCutFactor, int maxMajorCycle, float lambda, float alpha, int deconvIterations, float deconvEpsilon)
        {
            var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
            var psfVis   = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];

            for (int i = 0; i < data.Visibilities.GetLength(0); i++)
            {
                for (int j = 0; j < data.Visibilities.GetLength(1); j++)
                {
                    for (int k = 0; k < data.Visibilities.GetLength(2); k++)
                    {
                        if (!data.Flags[i, j, k])
                        {
                            psfVis[i, j, k] = new Complex(1.0, 0);
                        }
                        else
                        {
                            psfVis[i, j, k] = new Complex(0, 0);
                        }
                    }
                }
            }

            Console.WriteLine("gridding psf");
            var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies);
            var psf     = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);

            var totalWatch   = new Stopwatch();
            var currentWatch = new Stopwatch();

            var totalSize   = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var psfCut      = PSF.Cut(psf, psfCutFactor);
            var maxSidelobe = PSF.CalcMaxSidelobe(psf, psfCutFactor);

            IDeconvolver deconvolver = null;

            if (useGPU & GPUSerialCD.IsGPUSupported())
            {
                deconvolver = new GPUSerialCD(totalSize, psfCut, 1000);
            }
            else if (useGPU & !GPUSerialCD.IsGPUSupported())
            {
                Console.WriteLine("GPU not supported by library. Switching to CPU implementation");
                deconvolver = new FastSerialCD(totalSize, psfCut);
            }
            else
            {
                deconvolver = new FastSerialCD(totalSize, psfCut);
            }

            var psfBMap = psfCut;

            using (var gCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))))
                using (var gCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))))
                {
                    var currentGCalculator = gCalculator;
                    var maxLipschitz       = PSF.CalcMaxLipschitz(psfCut);
                    var lambdaLipschitz    = (float)(lambda * maxLipschitz);
                    var lambdaTrue         = (float)(lambda * PSF.CalcMaxLipschitz(psf));
                    var switchedToOtherPsf = false;

                    var xImage      = new float[c.GridSize, c.GridSize];
                    var residualVis = data.Visibilities;
                    DeconvolutionResult lastResult = null;
                    for (int cycle = 0; cycle < maxMajorCycle; cycle++)
                    {
                        Console.WriteLine("Beginning Major cycle " + cycle);
                        var dirtyGrid  = IDG.GridW(c, metadata, residualVis, data.UVW, data.Frequencies);
                        var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
                        FFT.Shift(dirtyImage);
                        FitsIO.Write(dirtyImage, obsName + "_dirty_serial_majorCycle" + cycle + ".fits");

                        currentWatch.Restart();
                        totalWatch.Start();
                        var maxDirty         = Residuals.GetMax(dirtyImage);
                        var gradients        = gCalculator.Convolve(dirtyImage);
                        var maxB             = Residuals.GetMax(gradients);
                        var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
                        var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                        var currentLambda    = (float)Math.Max(currentSideLobe / alpha, lambdaLipschitz);

                        var objective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);

                        var absMax = deconvolver.GetAbsMaxDiff(xImage, gradients, lambdaTrue, alpha);

                        if (absMax >= MAJOR_EPSILON)
                        {
                            lastResult = deconvolver.Deconvolve(xImage, gradients, currentLambda, alpha, deconvIterations, deconvEpsilon);
                        }

                        if (lambda == currentLambda & !switchedToOtherPsf)
                        {
                            currentGCalculator = gCalculator2;
                            lambda             = lambdaTrue;
                            maxLipschitz       = PSF.CalcMaxLipschitz(psf);
                            switchedToOtherPsf = true;
                        }

                        FitsIO.Write(xImage, obsName + "_model_serial_majorCycle" + cycle + ".fits");

                        currentWatch.Stop();
                        totalWatch.Stop();

                        if (absMax < MAJOR_EPSILON)
                        {
                            break;
                        }

                        FFT.Shift(xImage);
                        var xGrid = FFT.Forward(xImage);
                        FFT.Shift(xImage);
                        var modelVis = IDG.DeGridW(c, metadata, xGrid, data.UVW, data.Frequencies);
                        residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
                    }

                    Console.WriteLine("Reconstruction finished in (seconds): " + totalWatch.Elapsed.TotalSeconds);
                }
        }
Ejemplo n.º 6
0
        public static void GenerateCLEANExample(string simulatedLocation, string outputFolder)
        {
            var data     = MeasurementData.LoadSimulatedPoints(simulatedLocation);
            var cellSize = 1.0 / 3600.0 * Math.PI / 180.0;
            var c        = new GriddingConstants(data.VisibilitiesCount, 256, 8, 4, 512, (float)cellSize, 1, 0.0);
            var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);

            var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies);
            var psf     = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);

            Directory.CreateDirectory(outputFolder);
            var reconstruction = new float[c.GridSize, c.GridSize];

            var residualVis = data.Visibilities;

            for (int cycle = 0; cycle < 10; cycle++)
            {
                Console.WriteLine("in cycle " + cycle);
                var dirtyGrid  = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies);
                var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                //FitsIO.Write(dirtyImage, Path.Combine(outputFolder, "dirty_CLEAN_" + cycle + ".fits"));
                Tools.WriteToMeltCSV(dirtyImage, Path.Combine(outputFolder, "dirty_CLEAN_" + cycle + ".csv"));

                var maxY = -1;
                var maxX = -1;
                var max  = 0.0f;
                for (int y = 0; y < dirtyImage.GetLength(0); y++)
                {
                    for (int x = 0; x < dirtyImage.GetLength(1); x++)
                    {
                        if (max < Math.Abs(dirtyImage[y, x]))
                        {
                            maxY = y;
                            maxX = x;
                            max  = Math.Abs(dirtyImage[y, x]);
                        }
                    }
                }

                //FitsIO.Write(reconstruction, Path.Combine(outputFolder, "model_CLEAN_" + cycle + ".fits"));
                Tools.WriteToMeltCSV(PSF.Cut(reconstruction), Path.Combine(outputFolder, "model_CLEAN_" + cycle + ".csv"));

                reconstruction[maxY, maxX] += 0.5f * dirtyImage[maxY, maxX];

                FFT.Shift(reconstruction);
                var xGrid = FFT.Forward(reconstruction);
                FFT.Shift(reconstruction);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies);
                residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
            }

            var cleanbeam = new float[c.GridSize, c.GridSize];
            var x0        = c.GridSize / 2;
            var y0        = c.GridSize / 2;

            for (int y = 0; y < cleanbeam.GetLength(0); y++)
            {
                for (int x = 0; x < cleanbeam.GetLength(1); x++)
                {
                    cleanbeam[y, x] = (float)(1.0 * Math.Exp(-(Math.Pow(x0 - x, 2) / 16 + Math.Pow(y0 - y, 2) / 16)));
                }
            }

            FitsIO.Write(cleanbeam, Path.Combine(outputFolder, "clbeam.fits"));

            FFT.Shift(cleanbeam);
            var CL      = FFT.Forward(cleanbeam);
            var REC     = FFT.Forward(reconstruction);
            var CONF    = Common.Fourier2D.Multiply(REC, CL);
            var cleaned = FFT.BackwardFloat(CONF, reconstruction.Length);

            //FFT.Shift(cleaned);
            //FitsIO.Write(cleaned, Path.Combine(outputFolder, "rec_CLEAN.fits"));
            Tools.WriteToMeltCSV(PSF.Cut(cleaned), Path.Combine(outputFolder, "rec_CLEAN.csv"));
        }
Ejemplo n.º 7
0
        public static void DebugdWStack()
        {
            var    frequencies  = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\freq.fits");
            var    uvw          = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw0.fits");
            var    flags        = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length);
            double norm         = 2.0;
            var    visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);

            for (int i = 1; i < 8; i++)
            {
                var uvw0          = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw" + i + ".fits");
                var flags0        = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length);
                var visibilities0 = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, norm);
                uvw          = FitsIO.Stitch(uvw, uvw0);
                flags        = FitsIO.Stitch(flags, flags0);
                visibilities = FitsIO.Stitch(visibilities, visibilities0);
            }

            var maxW = 0.0;

            for (int i = 0; i < uvw.GetLength(0); i++)
            {
                for (int j = 0; j < uvw.GetLength(1); j++)
                {
                    maxW = Math.Max(maxW, Math.Abs(uvw[i, j, 2]));
                }
            }
            maxW = Partitioner.MetersToLambda(maxW, frequencies[frequencies.Length - 1]);

            var visCount2 = 0;

            for (int i = 0; i < flags.GetLength(0); i++)
            {
                for (int j = 0; j < flags.GetLength(1); j++)
                {
                    for (int k = 0; k < flags.GetLength(2); k++)
                    {
                        if (!flags[i, j, k])
                        {
                            visCount2++;
                        }
                    }
                }
            }
            var    visibilitiesCount = visCount2;
            int    gridSize          = 4096;
            int    subgridsize       = 16;
            int    kernelSize        = 8;
            int    max_nr_timesteps  = 1024;
            double cellSize          = 1.6 / 3600.0 * PI / 180.0;
            int    wLayerCount       = 32;
            double wStep             = maxW / (wLayerCount);
            var    c        = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep);
            var    c2       = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0);
            var    metadata = Partitioner.CreatePartition(c, uvw, frequencies);

            var psfVis = new Complex[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length];

            for (int i = 0; i < visibilities.GetLength(0); i++)
            {
                for (int j = 0; j < visibilities.GetLength(1); j++)
                {
                    for (int k = 0; k < visibilities.GetLength(2); k++)
                    {
                        if (!flags[i, j, k])
                        {
                            psfVis[i, j, k] = new Complex(1.0, 0);
                        }
                        else
                        {
                            psfVis[i, j, k] = new Complex(0, 0);
                        }
                    }
                }
            }

            var psfGrid = IDG.GridW(c, metadata, psfVis, uvw, frequencies);
            var psf     = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);

            FitsIO.Write(psf, "psfWStack.fits");

            var totalSize      = new Rectangle(0, 0, gridSize, gridSize);
            var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)));
            var fastCD         = new FastSerialCD(totalSize, psf);
            var lambda         = 0.4f * fastCD.MaxLipschitz;
            var alpha          = 0.1f;

            var xImage      = new float[gridSize, gridSize];
            var residualVis = visibilities;

            for (int cycle = 0; cycle < 8; cycle++)
            {
                var dirtyGrid = IDG.GridW(c, metadata, residualVis, uvw, frequencies);
                var dirty     = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
                FFT.Shift(dirty);

                FitsIO.Write(dirty, "dirty_" + cycle + ".fits");
                bMapCalculator.ConvolveInPlace(dirty);
                FitsIO.Write(dirty, "bMap_" + cycle + ".fits");
                var result = fastCD.Deconvolve(xImage, dirty, lambda, alpha, 10000, 1e-4f);

                FitsIO.Write(xImage, "xImageGreedy" + cycle + ".fits");

                FFT.Shift(xImage);
                var xGrid = FFT.Forward(xImage);
                FFT.Shift(xImage);
                var modelVis  = IDG.DeGridW(c, metadata, xGrid, uvw, frequencies);
                var modelGrid = IDG.GridW(c, metadata, modelVis, uvw, frequencies);
                var model     = FFT.WStackIFFTFloat(modelGrid, c.VisibilitiesCount);
                FFT.Shift(model);
                FitsIO.Write(model, "model_" + cycle + ".fits");
                residualVis = Visibilities.Substract(visibilities, modelVis, flags);
            }
        }
        private static void ReconstructPCDM(MeasurementData input, GriddingConstants c, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, int processorCount)
        {
            var totalWatch   = new Stopwatch();
            var currentWatch = new Stopwatch();

            var totalSize    = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var psfCut       = PSF.Cut(fullPsf, CUT_FACTOR_PCDM);
            var maxSidelobe  = PSF.CalcMaxSidelobe(fullPsf, CUT_FACTOR_PCDM);
            var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2);
            var random       = new Random(123);
            var pcdm         = new ParallelCoordinateDescent(totalSize, psfCut, 1, 1000, searchPercent);

            var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);

            using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
                using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
                    using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf))
                    {
                        var currentBMapCalculator = bMapCalculator;

                        var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
                        var lambda       = (float)(LAMBDA * maxLipschitz);
                        var lambdaTrue   = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
                        var alpha        = ALPHA;

                        var switchedToOtherPsf = false;
                        var writer             = new StreamWriter(folder + "/" + file + ".txt");
                        var xImage             = new float[c.GridSize, c.GridSize];
                        var residualVis        = input.Visibilities;
                        ParallelCoordinateDescent.PCDMStatistics lastResult = null;
                        for (int cycle = 0; cycle < 6; cycle++)
                        {
                            Console.WriteLine("Beginning Major cycle " + cycle);
                            var dirtyGrid  = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
                            var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
                            FFT.Shift(dirtyImage);
                            FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");

                            currentWatch.Restart();
                            totalWatch.Start();

                            var breakMajor       = false;
                            var minLambda        = 0.0f;
                            var dirtyCopy        = Copy(dirtyImage);
                            var xCopy            = Copy(xImage);
                            var currentLambda    = 0f;
                            var currentObjective = 0.0;
                            var absMax           = 0.0f;
                            for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++)
                            {
                                Console.WriteLine("Beginning Minor Cycle " + minorCycle);
                                var maxDirty         = Residuals.GetMax(dirtyImage);
                                var bMap             = currentBMapCalculator.Convolve(dirtyImage);
                                var maxB             = Residuals.GetMax(bMap);
                                var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
                                var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                                currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);

                                if (minorCycle == 0)
                                {
                                    minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
                                }
                                if (currentLambda < minLambda)
                                {
                                    currentLambda = minLambda;
                                }
                                currentObjective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
                                absMax           = pcdm.GetAbsMax(xImage, bMap, lambdaTrue, alpha);
                                if (absMax < MAJOR_STOP)
                                {
                                    breakMajor = true;
                                    break;
                                }

                                lastResult = pcdm.Deconvolve(xImage, bMap, currentLambda, alpha, 100, 1e-5f);

                                if (currentLambda == lambda | currentLambda == minLambda)
                                {
                                    break;
                                }

                                var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
                                Parallel.For(0, xCopy.GetLength(0), (i) =>
                                {
                                    for (int j = 0; j < xCopy.GetLength(1); j++)
                                    {
                                        residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
                                    }
                                });
                                residualsConvolver.ConvolveInPlace(residualsUpdate);
                                Parallel.For(0, xCopy.GetLength(0), (i) =>
                                {
                                    for (int j = 0; j < xCopy.GetLength(1); j++)
                                    {
                                        dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
                                    }
                                });
                            }

                            currentWatch.Stop();
                            totalWatch.Stop();
                            writer.WriteLine(cycle + ";" + currentLambda + ";" + currentObjective + ";" + absMax + ";" + lastResult.IterationCount + ";" + totalWatch.Elapsed.TotalSeconds + ";" + currentWatch.Elapsed.TotalSeconds);
                            writer.Flush();

                            FitsIO.Write(xImage, folder + "/xImage_pcdm_" + cycle + ".fits");

                            if (breakMajor)
                            {
                                break;
                            }
                            if (currentLambda == lambda & !switchedToOtherPsf)
                            {
                                pcdm.ResetAMap(fullPsf);
                                currentBMapCalculator = bMapCalculator2;
                                lambda             = lambdaTrue;
                                switchedToOtherPsf = true;
                                writer.WriteLine("switched");
                                writer.Flush();
                            }

                            FFT.Shift(xImage);
                            var xGrid = FFT.Forward(xImage);
                            FFT.Shift(xImage);
                            var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
                            residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
                        }

                        writer.Close();
                    }
        }
Ejemplo n.º 9
0
        public static void DebugSimulatedApprox()
        {
            var    frequencies  = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits");
            var    uvw          = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits");
            var    flags        = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
            double norm         = 2.0;
            var    visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);

            var    visibilitiesCount = visibilities.Length;
            int    gridSize          = 256;
            int    subgridsize       = 8;
            int    kernelSize        = 4;
            int    max_nr_timesteps  = 1024;
            double cellSize          = 1.0 / 3600.0 * PI / 180.0;
            var    c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);

            var watchTotal     = new Stopwatch();
            var watchForward   = new Stopwatch();
            var watchBackwards = new Stopwatch();
            var watchDeconv    = new Stopwatch();

            watchTotal.Start();
            var metadata = Partitioner.CreatePartition(c, uvw, frequencies);

            var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
            var psf     = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);
            var psfCut = PSF.Cut(psf);

            FitsIO.Write(psfCut, "psfCut.fits");

            var random         = new Random(123);
            var totalSize      = new Rectangle(0, 0, gridSize, gridSize);
            var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
            var fastCD         = new FastSerialCD(totalSize, psfCut);
            //fastCD.ResetAMap(psf);
            var lambda  = 0.5f * fastCD.MaxLipschitz;
            var alpha   = 0.8f;
            var approx  = new ApproxParallel();
            var approx2 = new ApproxFast(totalSize, psfCut, 4, 8, 0f, 0.25f, false, true);

            var xImage      = new float[gridSize, gridSize];
            var residualVis = visibilities;

            /*var truth = new double[gridSize, gridSize];
             * truth[30, 30] = 1.0;
             * truth[35, 36] = 1.5;
             * var truthVis = IDG.ToVisibilities(c, metadata, truth, uvw, frequencies);
             * visibilities = truthVis;
             * var residualVis = truthVis;*/
            var data = new ApproxFast.TestingData(new StreamWriter("approxConvergence.txt"));

            for (int cycle = 0; cycle < 4; cycle++)
            {
                //FORWARD
                watchForward.Start();
                var dirtyGrid  = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
                var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                FitsIO.Write(dirtyImage, "dirty_" + cycle + ".fits");
                watchForward.Stop();

                //DECONVOLVE
                watchDeconv.Start();
                //approx.ISTAStep(xImage, dirtyImage, psf, lambda, alpha);
                //FitsIO.Write(xImage, "xIsta.fits");
                //FitsIO.Write(dirtyImage, "dirtyFista.fits");
                //bMapCalculator.ConvolveInPlace(dirtyImage);
                //FitsIO.Write(dirtyImage, "bMap_" + cycle + ".fits");
                //var result = fastCD.Deconvolve(xImage, dirtyImage, 0.5f * fastCD.MaxLipschitz, 0.8f, 1000, 1e-4f);
                //var converged = approx.DeconvolveActiveSet(xImage, dirtyImage, psfCut, lambda, alpha, random, 8, 1, 1);
                //var converged = approx.DeconvolveGreedy(xImage, dirtyImage, psfCut, lambda, alpha, random, 4, 4, 500);
                //var converged = approx.DeconvolveApprox(xImage, dirtyImage, psfCut, lambda, alpha, random, 1, threads, 500, 1e-4f, cycle == 0);

                approx2.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, psf, lambda, alpha, random, 10, 1e-4f);


                if (data.converged)
                {
                    Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
                }
                else
                {
                    Console.WriteLine("-------------------------------not converged----------------------");
                }
                FitsIO.Write(xImage, "xImageApprox_" + cycle + ".fits");
                watchDeconv.Stop();

                //BACKWARDS
                watchBackwards.Start();
                FFT.Shift(xImage);
                var xGrid = FFT.Forward(xImage);
                FFT.Shift(xImage);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
                residualVis = Visibilities.Substract(visibilities, modelVis, flags);
                watchBackwards.Stop();
            }


            var dirtyGridCheck = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
            var dirtyCheck     = FFT.Backward(dirtyGridCheck, c.VisibilitiesCount);

            FFT.Shift(dirtyCheck);

            var l2Penalty      = Residuals.CalcPenalty(ToFloatImage(dirtyCheck));
            var elasticPenalty = ElasticNet.CalcPenalty(xImage, (float)lambda, (float)alpha);
            var sum            = l2Penalty + elasticPenalty;

            data.writer.Close();
        }
Ejemplo n.º 10
0
        public static void DebugILGPU()
        {
            var    frequencies  = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits");
            var    uvw          = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits");
            var    flags        = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
            double norm         = 2.0;
            var    visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);

            var    visibilitiesCount = visibilities.Length;
            int    gridSize          = 256;
            int    subgridsize       = 8;
            int    kernelSize        = 4;
            int    max_nr_timesteps  = 1024;
            double cellSize          = 1.0 / 3600.0 * PI / 180.0;
            var    c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);

            var watchTotal     = new Stopwatch();
            var watchForward   = new Stopwatch();
            var watchBackwards = new Stopwatch();
            var watchDeconv    = new Stopwatch();

            watchTotal.Start();
            var metadata = Partitioner.CreatePartition(c, uvw, frequencies);

            var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
            var psf     = FFT.Backward(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);

            var psfCutDouble = CutImg(psf);
            var psfCut       = ToFloatImage(psfCutDouble);

            FitsIO.Write(psfCut, "psfCut.fits");


            var totalSize      = new Rectangle(0, 0, gridSize, gridSize);
            var imageSection   = new Rectangle(0, 128, gridSize, gridSize);
            var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
            var fastCD         = new FastSerialCD(totalSize, psfCut);

            fastCD.ResetLipschitzMap(ToFloatImage(psf));
            var gpuCD  = new GPUSerialCD(totalSize, psfCut, 100);
            var lambda = 0.5f * fastCD.MaxLipschitz;
            var alpha  = 0.8f;

            var xImage      = new float[gridSize, gridSize];
            var residualVis = visibilities;

            /*var truth = new double[gridSize, gridSize];
             * truth[30, 30] = 1.0;
             * truth[35, 36] = 1.5;
             * var truthVis = IDG.ToVisibilities(c, metadata, truth, uvw, frequencies);
             * visibilities = truthVis;
             * var residualVis = truthVis;*/
            for (int cycle = 0; cycle < 4; cycle++)
            {
                //FORWARD
                watchForward.Start();
                var dirtyGrid  = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
                var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                FitsIO.Write(dirtyImage, "dirty_" + cycle + ".fits");
                watchForward.Stop();

                //DECONVOLVE
                watchDeconv.Start();
                bMapCalculator.ConvolveInPlace(dirtyImage);
                FitsIO.Write(dirtyImage, "bMap_" + cycle + ".fits");
                //var result = fastCD.Deconvolve(xImage, dirtyImage, lambda, alpha, 1000, 1e-4f);
                var result = gpuCD.Deconvolve(xImage, dirtyImage, lambda, alpha, 1000, 1e-4f);

                if (result.Converged)
                {
                    Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
                }
                else
                {
                    Console.WriteLine("-------------------------------not converged----------------------");
                }
                FitsIO.Write(xImage, "xImageGreedy" + cycle + ".fits");
                FitsIO.Write(dirtyImage, "residualDebug_" + cycle + ".fits");
                watchDeconv.Stop();

                //BACKWARDS
                watchBackwards.Start();
                FFT.Shift(xImage);
                var xGrid = FFT.Forward(xImage);
                FFT.Shift(xImage);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
                residualVis = Visibilities.Substract(visibilities, modelVis, flags);
                watchBackwards.Stop();

                var hello = FFT.Forward(xImage, 1.0);
                hello = Common.Fourier2D.Multiply(hello, psfGrid);
                var hImg = FFT.Backward(hello, (double)(128 * 128));
                //FFT.Shift(hImg);
                FitsIO.Write(hImg, "modelDirty_FFT.fits");

                var imgRec = IDG.ToImage(c, metadata, modelVis, uvw, frequencies);
                FitsIO.Write(imgRec, "modelDirty" + cycle + ".fits");
            }
        }
Ejemplo n.º 11
0
        public static void MeerKATFull()
        {
            var    frequencies  = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\freq.fits");
            var    uvw          = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw0.fits");
            var    flags        = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length);
            double norm         = 2.0;
            var    visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);

            for (int i = 1; i < 8; i++)
            {
                var uvw0          = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw" + i + ".fits");
                var flags0        = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length);
                var visibilities0 = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, norm);
                uvw          = FitsIO.Stitch(uvw, uvw0);
                flags        = FitsIO.Stitch(flags, flags0);
                visibilities = FitsIO.Stitch(visibilities, visibilities0);
            }

            /*
             * var frequencies = FitsIO.ReadFrequencies(@"freq.fits");
             * var uvw = FitsIO.ReadUVW("uvw0.fits");
             * var flags = FitsIO.ReadFlags("flags0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length);
             * double norm = 2.0;
             * var visibilities = FitsIO.ReadVisibilities("vis0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
             */
            var visCount2 = 0;

            for (int i = 0; i < flags.GetLength(0); i++)
            {
                for (int j = 0; j < flags.GetLength(1); j++)
                {
                    for (int k = 0; k < flags.GetLength(2); k++)
                    {
                        if (!flags[i, j, k])
                        {
                            visCount2++;
                        }
                    }
                }
            }
            var visibilitiesCount = visCount2;

            int gridSize    = 1024;
            int subgridsize = 16;
            int kernelSize  = 4;
            //cell = image / grid
            int    max_nr_timesteps = 512;
            double scaleArcSec      = 2.5 / 3600.0 * PI / 180.0;

            var watchTotal     = new Stopwatch();
            var watchForward   = new Stopwatch();
            var watchBackwards = new Stopwatch();
            var watchDeconv    = new Stopwatch();

            watchTotal.Start();

            var c        = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)scaleArcSec, 1, 0.0f);
            var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
            var psf      = IDG.CalculatePSF(c, metadata, uvw, flags, frequencies);

            FitsIO.Write(psf, "psf.fits");
            var psfCut = CutImg(psf, 2);

            FitsIO.Write(psfCut, "psfCut.fits");
            var maxSidelobe   = CommonDeprecated.PSF.CalcMaxSidelobe(psf);
            var psfCorrelated = CommonDeprecated.PSF.CalculateFourierCorrelation(psfCut, c.GridSize, c.GridSize);

            var xImage      = new double[gridSize, gridSize];
            var residualVis = visibilities;
            var maxCycle    = 2;

            for (int cycle = 0; cycle < maxCycle; cycle++)
            {
                watchForward.Start();
                var dirtyImage = IDG.ToImage(c, metadata, residualVis, uvw, frequencies);
                watchForward.Stop();
                FitsIO.Write(dirtyImage, "dirty" + cycle + ".fits");

                watchDeconv.Start();
                var sideLobe = maxSidelobe * GetMax(dirtyImage);
                Console.WriteLine("sideLobeLevel: " + sideLobe);
                var b             = CommonDeprecated.Residuals.CalculateBMap(dirtyImage, psfCorrelated, psfCut.GetLength(0), psfCut.GetLength(1));
                var lambda        = 0.8;
                var alpha         = 0.05;
                var currentLambda = Math.Max(1.0 / alpha * sideLobe, lambda);
                var converged     = SerialCDReference.DeconvolvePath(xImage, b, psfCut, currentLambda, 4.0, alpha, 5, 1000, 2e-5);
                //var converged = GreedyCD2.Deconvolve(xImage, b, psfCut, currentLambda, alpha, 5000);
                if (converged)
                {
                    Console.WriteLine("-----------------------------CONVERGED!!!! with lambda " + currentLambda + "------------------------");
                }
                else
                {
                    Console.WriteLine("-------------------------------not converged with lambda " + currentLambda + "----------------------");
                }

                watchDeconv.Stop();
                FitsIO.Write(xImage, "xImage_" + cycle + ".fits");

                watchBackwards.Start();
                FFT.Shift(xImage);
                var xGrid = FFT.Forward(xImage);
                FFT.Shift(xImage);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
                residualVis = Visibilities.Substract(visibilities, modelVis, flags);
                watchBackwards.Stop();
            }
            watchBackwards.Stop();
            watchTotal.Stop();

            var timetable = "total elapsed: " + watchTotal.Elapsed;

            timetable += "\n" + "idg forward elapsed: " + watchForward.Elapsed;
            timetable += "\n" + "idg backwards elapsed: " + watchBackwards.Elapsed;
            timetable += "\n" + "devonvolution: " + watchDeconv.Elapsed;
            File.WriteAllText("watches_single.txt", timetable);
        }
Ejemplo n.º 12
0
        public static void DebugSimulatedMixed()
        {
            var    frequencies       = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_mixed\freq.fits");
            var    uvw               = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_mixed\uvw.fits");
            var    flags             = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
            double norm              = 2.0;
            var    visibilities      = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_mixed\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
            var    visibilitiesCount = visibilities.Length;

            int gridSize    = 1024;
            int subgridsize = 16;
            int kernelSize  = 4;
            //cell = image / grid
            int    max_nr_timesteps = 512;
            double scaleArcSec      = 0.5 / 3600.0 * PI / 180.0;

            var watchTotal     = new Stopwatch();
            var watchForward   = new Stopwatch();
            var watchBackwards = new Stopwatch();
            var watchDeconv    = new Stopwatch();

            watchTotal.Start();

            var c        = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)scaleArcSec, 1, 0.0f);
            var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
            var psf      = IDG.CalculatePSF(c, metadata, uvw, flags, frequencies);

            FitsIO.Write(psf, "psf.fits");
            var psfCut = CutImg(psf, 2);

            FitsIO.Write(psfCut, "psfCut.fits");
            var maxSidelobe = CommonDeprecated.PSF.CalcMaxSidelobe(psf);

            var xImage      = new double[gridSize, gridSize];
            var residualVis = visibilities;
            var maxCycle    = 10;

            for (int cycle = 0; cycle < maxCycle; cycle++)
            {
                watchForward.Start();
                var dirtyImage = IDG.ToImage(c, metadata, residualVis, uvw, frequencies);
                watchForward.Stop();
                FitsIO.Write(dirtyImage, "dirty" + cycle + ".fits");

                watchDeconv.Start();
                var sideLobe = maxSidelobe * GetMax(dirtyImage);
                Console.WriteLine("sideLobeLevel: " + sideLobe);
                var PsfCorrelation = CommonDeprecated.PSF.CalculateFourierCorrelation(psfCut, c.GridSize, c.GridSize);
                var b             = CommonDeprecated.Residuals.CalculateBMap(dirtyImage, PsfCorrelation, psfCut.GetLength(0), psfCut.GetLength(1));
                var lambda        = 100.0;
                var alpha         = 0.95;
                var currentLambda = Math.Max(1.0 / alpha * sideLobe, lambda);
                var converged     = SerialCDReference.DeconvolvePath(xImage, b, psfCut, currentLambda, 5.0, alpha, 5, 6000, 1e-3);
                //var converged = GreedyCD2.Deconvolve(xImage, b, psfCut, currentLambda, alpha, 5000);
                if (converged)
                {
                    Console.WriteLine("-----------------------------CONVERGED!!!! with lambda " + currentLambda + "------------------------");
                }
                else
                {
                    Console.WriteLine("-------------------------------not converged with lambda " + currentLambda + "----------------------");
                }

                watchDeconv.Stop();
                FitsIO.Write(xImage, "xImage_" + cycle + ".fits");

                watchBackwards.Start();
                FFT.Shift(xImage);
                var xGrid = FFT.Forward(xImage);
                FFT.Shift(xImage);
                var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
                residualVis = Visibilities.Substract(visibilities, modelVis, flags);
                watchBackwards.Stop();
            }
            watchBackwards.Stop();
            watchTotal.Stop();

            var timetable = "total elapsed: " + watchTotal.Elapsed;

            timetable += "\n" + "idg forward elapsed: " + watchForward.Elapsed;
            timetable += "\n" + "idg backwards elapsed: " + watchBackwards.Elapsed;
            timetable += "\n" + "devonvolution: " + watchDeconv.Elapsed;
            File.WriteAllText("watches_single.txt", timetable);
        }
Ejemplo n.º 13
0
        private static ReconstructionInfo Reconstruct(Data input, float fullLipschitz, float[,] maskedPsf, string folder, float maskFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon, bool maskPsf2)
        {
            var info      = new ReconstructionInfo();
            var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize);

            var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(maskedPsf, totalSize), new Rectangle(0, 0, maskedPsf.GetLength(0), maskedPsf.GetLength(1)));
            var maskedPsf2     = PSF.CalcPSFSquared(maskedPsf);

            if (maskPsf2)
            {
                Mask(maskedPsf2, 1e-5f);
            }
            writer.WriteLine((CountNonZero(maskedPsf2) - maskedPsf2.Length) / (double)maskedPsf2.Length);
            var fastCD = new FastSerialCD(totalSize, totalSize, maskedPsf, maskedPsf2);

            FitsIO.Write(maskedPsf, folder + maskFactor + "psf.fits");



            var lambda     = 0.4f * fastCD.MaxLipschitz;
            var lambdaTrue = 0.4f * fullLipschitz;
            var alpha      = 0.1f;

            var xImage      = new float[input.c.GridSize, input.c.GridSize];
            var residualVis = input.visibilities;
            DeconvolutionResult lastResult = null;

            for (int cycle = 0; cycle < maxMajor; cycle++)
            {
                Console.WriteLine("cycle " + cycle);
                var dirtyGrid  = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies);
                var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits");

                //calc data and reg penalty
                var dataPenalty       = Residuals.CalcPenalty(dirtyImage);
                var regPenalty        = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
                var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha);
                info.lastDataPenalty = dataPenalty;
                info.lastRegPenalty  = regPenalty;

                bMapCalculator.ConvolveInPlace(dirtyImage);
                //FitsIO.Write(dirtyImage, folder + dirtyPrefix + "bmap_" + cycle + ".fits");
                var currentLambda = lambda;

                writer.Write(cycle + ";" + currentLambda + ";" + Residuals.GetMax(dirtyImage) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";");
                writer.Flush();

                //check wether we can minimize the objective further with the current psf
                var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff;
                var minimumReached   = (lastResult != null && lastResult.IterationCount < 100 && lastResult.Converged);
                if (!objectiveReached & !minimumReached)
                {
                    info.totalDeconv.Start();
                    lastResult = fastCD.Deconvolve(xImage, dirtyImage, currentLambda, alpha, 50000, epsilon);
                    info.totalDeconv.Stop();

                    FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits");
                    writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n");
                    writer.Flush();

                    FFT.Shift(xImage);
                    var xGrid = FFT.Forward(xImage);
                    FFT.Shift(xImage);
                    var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies);
                    residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags);
                }
                else
                {
                    writer.Write(false + ";0;0");
                    writer.Flush();
                    break;
                }
            }

            return(info);
        }
        private static ReconstructionInfo ReconstructGradientApprox(Data input, float[,] fullPsf, string folder, int cutFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon)
        {
            var info            = new ReconstructionInfo();
            var psfCut          = PSF.Cut(fullPsf, cutFactor);
            var maxSidelobe     = PSF.CalcMaxSidelobe(fullPsf, cutFactor);
            var totalSize       = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize);
            var psfBMap         = psfCut;
            var bMapCalculator  = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1)));
            var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)));
            var fastCD          = new FastSerialCD(totalSize, psfCut);
            var fastCD2         = new FastSerialCD(totalSize, psfCut);

            fastCD2.ResetLipschitzMap(fullPsf);
            FitsIO.Write(psfCut, folder + cutFactor + "psf.fits");

            var lambda     = LAMBDA_GLOBAL * fastCD.MaxLipschitz;
            var lambdaTrue = (float)(LAMBDA_GLOBAL * PSF.CalcMaxLipschitz(fullPsf));

            var xImage      = new float[input.c.GridSize, input.c.GridSize];
            var residualVis = input.visibilities;
            DeconvolutionResult lastResult = null;
            var firstTimeConverged         = false;
            var lastLambda = 0.0f;

            for (int cycle = 0; cycle < maxMajor; cycle++)
            {
                Console.WriteLine("cycle " + cycle);
                var dirtyGrid  = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies);
                var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount);
                FFT.Shift(dirtyImage);
                FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits");

                //calc data and reg penalty
                var dataPenalty       = Residuals.CalcPenalty(dirtyImage);
                var regPenalty        = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
                var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha);
                info.lastDataPenalty = dataPenalty;
                info.lastRegPenalty  = regPenalty;

                var maxDirty = Residuals.GetMax(dirtyImage);
                var bMap     = bMapCalculator.Convolve(dirtyImage);
                FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + ".fits");
                var maxB             = Residuals.GetMax(bMap);
                var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f);
                var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                var currentLambda    = Math.Max(currentSideLobe / alpha, lambda);

                writer.Write(cycle + ";" + currentLambda + ";" + currentSideLobe + ";" + ";" + fastCD2.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";");;
                writer.Flush();

                //check wether we can minimize the objective further with the current psf
                var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff;
                var minimumReached   = (lastResult != null && lastResult.Converged && fastCD2.GetAbsMaxDiff(xImage, dirtyImage, lambdaTrue, alpha) < MAJOR_EPSILON && currentLambda == lambda);
                if (lambda == lastLambda & !firstTimeConverged)
                {
                    firstTimeConverged = true;
                    minimumReached     = false;
                }

                if (!objectiveReached & !minimumReached)
                {
                    //writer.Write(firstTimeConverged + ";");
                    //writer.Flush();
                    info.totalDeconv.Start();
                    if (!firstTimeConverged)
                    {
                        lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon);
                    }
                    else
                    {
                        bMap = bMapCalculator2.Convolve(dirtyImage);
                        //FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + "_full.fits");
                        maxB             = Residuals.GetMax(bMap);
                        correctionFactor = Math.Max(maxB / (maxDirty * fastCD2.MaxLipschitz), 1.0f);
                        currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                        currentLambda    = Math.Max(currentSideLobe / alpha, lambdaTrue);
                        info.totalDeconv.Start();
                        lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon);
                        info.totalDeconv.Stop();
                    }

                    info.totalDeconv.Stop();

                    FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits");
                    writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n");
                    writer.Flush();

                    FFT.Shift(xImage);
                    var xGrid = FFT.Forward(xImage);
                    FFT.Shift(xImage);
                    var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies);
                    residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags);
                }
                else
                {
                    writer.Write(false + ";0;0\n");
                    writer.Flush();

                    break;
                }

                lastLambda = currentLambda;
            }

            bMapCalculator.Dispose();
            bMapCalculator2.Dispose();

            return(info);
        }
Ejemplo n.º 15
0
        /// <summary>
        /// Major cycle implemnentation for the parallel coordinate descent algorithm
        /// </summary>
        /// <param name="data"></param>
        /// <param name="c"></param>
        /// <param name="psfCutFactor"></param>
        /// <param name="maxMajorCycle"></param>
        /// <param name="maxMinorCycle"></param>
        /// <param name="lambda"></param>
        /// <param name="alpha"></param>
        /// <param name="deconvIterations"></param>
        /// <param name="deconvEpsilon"></param>
        public static void ReconstructPCDM(string obsName, MeasurementData data, GriddingConstants c, int psfCutFactor, int maxMajorCycle, int maxMinorCycle, float lambda, float alpha, int deconvIterations, float deconvEpsilon)
        {
            var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
            var psfVis   = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];

            for (int i = 0; i < data.Visibilities.GetLength(0); i++)
            {
                for (int j = 0; j < data.Visibilities.GetLength(1); j++)
                {
                    for (int k = 0; k < data.Visibilities.GetLength(2); k++)
                    {
                        if (!data.Flags[i, j, k])
                        {
                            psfVis[i, j, k] = new Complex(1.0, 0);
                        }
                        else
                        {
                            psfVis[i, j, k] = new Complex(0, 0);
                        }
                    }
                }
            }

            Console.WriteLine("gridding psf");
            var psfGrid = IDG.Grid(c, metadata, psfVis, data.UVW, data.Frequencies);
            var psf     = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);

            FFT.Shift(psf);

            var totalWatch   = new Stopwatch();
            var currentWatch = new Stopwatch();

            var totalSize    = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var psfCut       = PSF.Cut(psf, psfCutFactor);
            var maxSidelobe  = PSF.CalcMaxSidelobe(psf, psfCutFactor);
            var sidelobeHalf = PSF.CalcMaxSidelobe(psf, 2);

            var pcdm = new ParallelCoordinateDescent(totalSize, psfCut, Environment.ProcessorCount, 1000);

            using (var gCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
                using (var gCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))))
                    using (var residualsConvolver = new PaddedConvolver(totalSize, psf))
                    {
                        var currentGCalculator = gCalculator;

                        var maxLipschitz    = PSF.CalcMaxLipschitz(psfCut);
                        var lambdaLipschitz = (float)(lambda * maxLipschitz);
                        var lambdaTrue      = (float)(lambda * PSF.CalcMaxLipschitz(psf));

                        var switchedToOtherPsf = false;
                        var xImage             = new float[c.GridSize, c.GridSize];
                        var residualVis        = data.Visibilities;
                        ParallelCoordinateDescent.PCDMStatistics lastResult = null;
                        for (int cycle = 0; cycle < maxMajorCycle; cycle++)
                        {
                            Console.WriteLine("Beginning Major cycle " + cycle);
                            var dirtyGrid  = IDG.GridW(c, metadata, residualVis, data.UVW, data.Frequencies);
                            var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
                            FFT.Shift(dirtyImage);
                            FitsIO.Write(dirtyImage, obsName + "_dirty_pcdm_majorCycle" + cycle + ".fits");

                            currentWatch.Restart();
                            totalWatch.Start();

                            var breakMajor       = false;
                            var minLambda        = 0.0f;
                            var dirtyCopy        = Copy(dirtyImage);
                            var xCopy            = Copy(xImage);
                            var currentLambda    = 0f;
                            var currentObjective = 0.0;
                            var absMax           = 0.0f;
                            for (int minorCycle = 0; minorCycle < maxMinorCycle; minorCycle++)
                            {
                                Console.WriteLine("Beginning Minor Cycle " + minorCycle);
                                var maxDirty         = Residuals.GetMax(dirtyImage);
                                var bMap             = currentGCalculator.Convolve(dirtyImage);
                                var maxB             = Residuals.GetMax(bMap);
                                var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
                                var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                                currentLambda = (float)Math.Max(currentSideLobe / alpha, lambdaLipschitz);

                                if (minorCycle == 0)
                                {
                                    minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
                                }
                                if (currentLambda < minLambda)
                                {
                                    currentLambda = minLambda;
                                }
                                currentObjective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
                                absMax           = pcdm.GetAbsMax(xImage, bMap, lambdaTrue, alpha);
                                if (absMax < MAJOR_EPSILON)
                                {
                                    breakMajor = true;
                                    break;
                                }

                                lastResult = pcdm.Deconvolve(xImage, bMap, currentLambda, alpha, 40, deconvEpsilon);

                                if (currentLambda == lambda | currentLambda == minLambda)
                                {
                                    break;
                                }

                                var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
                                Parallel.For(0, xCopy.GetLength(0), (i) =>
                                {
                                    for (int j = 0; j < xCopy.GetLength(1); j++)
                                    {
                                        residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
                                    }
                                });
                                residualsConvolver.ConvolveInPlace(residualsUpdate);
                                Parallel.For(0, xCopy.GetLength(0), (i) =>
                                {
                                    for (int j = 0; j < xCopy.GetLength(1); j++)
                                    {
                                        dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
                                    }
                                });
                            }

                            currentWatch.Stop();
                            totalWatch.Stop();

                            if (breakMajor)
                            {
                                break;
                            }
                            if (currentLambda == lambda & !switchedToOtherPsf)
                            {
                                pcdm.ResetAMap(psf);
                                currentGCalculator = gCalculator2;
                                lambda             = lambdaTrue;
                                switchedToOtherPsf = true;
                            }

                            FitsIO.Write(xImage, obsName + "_model_pcdm_majorCycle" + cycle + ".fits");

                            FFT.Shift(xImage);
                            var xGrid = FFT.Forward(xImage);
                            FFT.Shift(xImage);
                            var modelVis = IDG.DeGridW(c, metadata, xGrid, data.UVW, data.Frequencies);
                            residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
                        }

                        Console.WriteLine("Reconstruction finished in (seconds): " + totalWatch.Elapsed.TotalSeconds);
                    }
        }
        public static float[,] Reconstruct(Intracommunicator comm, DistributedData.LocalDataset local, GriddingConstants c, int maxCycle, float lambda, float alpha, int iterPerCycle = 1000, bool usePathDeconvolution = false)
        {
            var watchTotal    = new Stopwatch();
            var watchForward  = new Stopwatch();
            var watchBackward = new Stopwatch();
            var watchDeconv   = new Stopwatch();

            watchTotal.Start();

            var metadata = Partitioner.CreatePartition(c, local.UVW, local.Frequencies);

            var patchSize = CalculateLocalImageSection(comm.Rank, comm.Size, c.GridSize, c.GridSize);
            var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);

            //calculate psf and prepare for correlation in the Fourier space
            var psf = CalculatePSF(comm, c, metadata, local.UVW, local.Flags, local.Frequencies);

            Complex[,] PsfCorrelation = null;
            var maxSidelobe = PSF.CalcMaxSidelobe(psf);

            lambda = (float)(lambda * PSF.CalcMaxLipschitz(psf));

            StreamWriter writer = null;

            if (comm.Rank == 0)
            {
                FitsIO.Write(psf, "psf.fits");
                Console.WriteLine("done PSF gridding ");
                PsfCorrelation = PSF.CalcPaddedFourierCorrelation(psf, totalSize);
                writer         = new StreamWriter(comm.Size + "runtimestats.txt");
            }

            var deconvovler = new MPIGreedyCD(comm, totalSize, patchSize, psf);

            var residualVis = local.Visibilities;
            var xLocal      = new float[patchSize.YEnd - patchSize.Y, patchSize.XEnd - patchSize.X];


            for (int cycle = 0; cycle < maxCycle; cycle++)
            {
                if (comm.Rank == 0)
                {
                    Console.WriteLine("cycle " + cycle);
                }
                var dirtyImage = ForwardCalculateB(comm, c, metadata, residualVis, local.UVW, local.Frequencies, PsfCorrelation, psf, maxSidelobe, watchForward);

                var bLocal = GetImgSection(dirtyImage.Image, patchSize);

                MPIGreedyCD.Statistics lastRun;
                if (usePathDeconvolution)
                {
                    var currentLambda = Math.Max(1.0f / alpha * dirtyImage.MaxSidelobeLevel, lambda);
                    lastRun = deconvovler.DeconvolvePath(xLocal, bLocal, currentLambda, 4.0f, alpha, 5, iterPerCycle, 2e-5f);
                }
                else
                {
                    lastRun = deconvovler.Deconvolve(xLocal, bLocal, lambda, alpha, iterPerCycle, 1e-5f);
                }

                if (comm.Rank == 0)
                {
                    WriteToFile(cycle, lastRun, writer);
                    if (lastRun.Converged)
                    {
                        Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
                    }
                    else
                    {
                        Console.WriteLine("-------------------------------not converged----------------------");
                    }
                }
                comm.Barrier();
                if (comm.Rank == 0)
                {
                    watchDeconv.Stop();
                }

                float[][,] totalX = null;
                comm.Gather(xLocal, 0, ref totalX);
                Complex[,] modelGrid = null;
                if (comm.Rank == 0)
                {
                    watchBackward.Start();
                    var x = new float[c.GridSize, c.GridSize];
                    StitchImage(totalX, x, comm.Size);
                    FitsIO.Write(x, "xImage_" + cycle + ".fits");
                    FFT.Shift(x);
                    modelGrid = FFT.Forward(x);
                }
                comm.Broadcast(ref modelGrid, 0);

                var modelVis = IDG.DeGrid(c, metadata, modelGrid, local.UVW, local.Frequencies);
                residualVis = Visibilities.Substract(local.Visibilities, modelVis, local.Flags);
            }
            writer.Close();

            float[][,] gatherX = null;
            comm.Gather(xLocal, 0, ref gatherX);
            float[,] reconstructed = null;
            if (comm.Rank == 0)
            {
                reconstructed = new float[c.GridSize, c.GridSize];;
                StitchImage(gatherX, reconstructed, comm.Size);
            }

            return(reconstructed);
        }
        private static void ReconstructSerial(MeasurementData input, GriddingConstants c, float[,] fullPsf, string folder, string file, int processorCount)
        {
            var totalWatch   = new Stopwatch();
            var currentWatch = new Stopwatch();

            var totalSize   = new Rectangle(0, 0, c.GridSize, c.GridSize);
            var psfCut      = PSF.Cut(fullPsf, CUT_FACTOR_SERIAL);
            var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, CUT_FACTOR_SERIAL);
            var fastCD      = new FastSerialCD(totalSize, psfCut, processorCount);
            var metadata    = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);

            var writer  = new StreamWriter(folder + "/" + file + ".txt");
            var psfBMap = psfCut;

            using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))))
                using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
                {
                    var currentBMapCalculator = bMapCalculator;

                    var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
                    var lambda       = (float)(LAMBDA * maxLipschitz);
                    var lambdaTrue   = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
                    var alpha        = ALPHA;

                    var switchedToOtherPsf         = false;
                    var xImage                     = new float[c.GridSize, c.GridSize];
                    var residualVis                = input.Visibilities;
                    DeconvolutionResult lastResult = null;
                    for (int cycle = 0; cycle < 6; cycle++)
                    {
                        Console.WriteLine("cycle " + cycle);
                        var dirtyGrid  = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
                        var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
                        FFT.Shift(dirtyImage);
                        FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");

                        currentWatch.Restart();
                        totalWatch.Start();
                        var maxDirty         = Residuals.GetMax(dirtyImage);
                        var bMap             = bMapCalculator.Convolve(dirtyImage);
                        var maxB             = Residuals.GetMax(bMap);
                        var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f);
                        var currentSideLobe  = maxB * maxSidelobe * correctionFactor;
                        var currentLambda    = Math.Max(currentSideLobe / alpha, lambda);


                        var objective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);

                        var absMax = fastCD.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha);

                        if (absMax >= MAJOR_STOP)
                        {
                            lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, 1e-5f);
                        }

                        if (lambda == currentLambda & !switchedToOtherPsf)
                        {
                            currentBMapCalculator = bMapCalculator2;
                            lambda             = lambdaTrue;
                            switchedToOtherPsf = true;
                        }

                        currentWatch.Stop();
                        totalWatch.Stop();
                        writer.WriteLine(cycle + ";" + currentLambda + ";" + objective + ";" + absMax + ";" + lastResult.IterationCount + ";" + totalWatch.Elapsed.TotalSeconds + ";" + currentWatch.Elapsed.TotalSeconds);
                        writer.Flush();

                        if (absMax < MAJOR_STOP)
                        {
                            break;
                        }

                        FFT.Shift(xImage);
                        var xGrid = FFT.Forward(xImage);
                        FFT.Shift(xImage);
                        var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
                        residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
                    }
                }
        }