public GPUSerialCD(Rectangle totalSize, float[,] psf, float[,] psfSquared, int nrBatchIterations) { this.totalSize = totalSize; psf2 = psfSquared; aMap = PSF.CalcAMap(psf, totalSize); MaxLipschitz = Residuals.GetMax(psfSquared); batchIterations = nrBatchIterations; c = new Context(ContextFlags.FastMath); var gpuIds = Accelerator.Accelerators.Where(id => id.AcceleratorType != AcceleratorType.CPU); if (gpuIds.Any()) { RunsOnGPU = true; accelerator = new CudaAccelerator(c, gpuIds.First().DeviceId); } else { Console.WriteLine("GPU vendor not supported. ILGPU switches to a !!!!VERY!!!! slow CPU implementation"); RunsOnGPU = false; accelerator = new CPUAccelerator(c, 4); } shrink = accelerator.LoadAutoGroupedStreamKernel <Index2, ArrayView2D <float>, ArrayView2D <float>, ArrayView2D <float>, ArrayView <float>, ArrayView <Pixel> >(ShrinkKernel); updateX = accelerator.LoadAutoGroupedStreamKernel <ILGPU.Index, ArrayView2D <float>, ArrayView <Pixel> >(UpdateXKernel); updateB = accelerator.LoadAutoGroupedStreamKernel <Index2, ArrayView2D <float>, ArrayView2D <float>, ArrayView <Pixel> >(UpdateBKernel); }
private static DirtyImage ForwardCalculateB(Intracommunicator comm, GriddingConstants c, List <List <Subgrid> > metadata, Complex[,,] visibilities, double[,,] uvw, double[] frequencies, Complex[,] PsfCorrelation, float[,] psfCut, float maxSidelobe, Stopwatch watchIdg) { Stopwatch another = new Stopwatch(); comm.Barrier(); if (comm.Rank == 0) { watchIdg.Start(); } var localGrid = IDG.Grid(c, metadata, visibilities, uvw, frequencies); float[,] image = null; float maxSideLobeLevel = 0.0f; var grid_total = comm.Reduce <Complex[, ]>(localGrid, SequentialSum, 0); if (comm.Rank == 0) { var dirtyImage = FFT.BackwardFloat(grid_total, c.VisibilitiesCount); FFT.Shift(dirtyImage); if (comm.Rank == 0) { FitsIO.Write(dirtyImage, "dirtyImage.fits"); } maxSideLobeLevel = maxSidelobe * Residuals.GetMax(dirtyImage); //remove spheroidal image = Residuals.CalcGradientMap(dirtyImage, PsfCorrelation, new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); watchIdg.Stop(); } comm.Broadcast(ref maxSideLobeLevel, 0); comm.Broadcast(ref image, 0); return(new DirtyImage(image, maxSideLobeLevel)); }
private static void ReconstructRandom(MeasurementData input, GriddingConstants c, float[,] psf, int blockSize, int iterCount, string file) { var cutFactor = 8; var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(psf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(psf, cutFactor); var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut)); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(psf)); var alpha = ALPHA; ApproxFast.LAMBDA_TEST = lambdaTrue; ApproxFast.ALPHA_TEST = alpha; var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies); var random = new Random(123); var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.0f, 0.0f, false, true, false); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); var data = new ApproxFast.TestingData(new StreamWriter(file + "_tmp.txt")); var xImage = new float[c.GridSize, c.GridSize]; var xCorr = Copy(xImage); var residualVis = input.Visibilities; var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); var gCorr = new float[c.GridSize, c.GridSize]; var shared = new ApproxFast.SharedData(currentLambda, alpha, 1, 1, 8, CountNonZero(psfCut), approx.psf2, approx.aMap, xImage, xCorr, bMap, gCorr, new Random()); shared.ActiveSet = ApproxFast.GetActiveSet(xImage, bMap, shared.YBlockSize, shared.XBlockSize, lambda, alpha, shared.AMap); shared.BlockLock = new int[shared.ActiveSet.Count]; shared.maxLipschitz = (float)PSF.CalcMaxLipschitz(psfCut); shared.MaxConcurrentIterations = 1000; approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, 1, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf); var output = Tools.LMC.CutN132Remnant(xImage); Tools.WriteToMeltCSV(output.Item1, file + "_1k.csv", output.Item2, output.Item3); FitsIO.Write(output.Item1, file + "_1k.fits"); FitsIO.Write(xImage, file + "_1k2.fits"); approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, iterCount, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf); output = Tools.LMC.CutN132Remnant(xImage); Tools.WriteToMeltCSV(output.Item1, file + "_10k.csv", output.Item2, output.Item3); FitsIO.Write(output.Item1, file + "_10k.fits"); FitsIO.Write(xImage, file + "_10k2.fits"); }
public static Complex[,] CalcPaddedFourierConvolution(double[,] psf, int yPadding, int xPadding) { var psfPadded = Residuals.Pad(psf, yPadding, xPadding); FFT.Shift(psfPadded); var PSFPadded = FFT.Forward(psfPadded, 1.0); return(PSFPadded); }
public FastSerialCD(Rectangle totalSize, Rectangle patchSize, float[,] psf, float[,] psfSquared, int processorLimit = -1) { this.patch = patchSize; psf2 = psfSquared; aMap = PSF.CalcAMap(psf, totalSize, patchSize); MaxLipschitz = Residuals.GetMax(psfSquared); parallelOptions = new ParallelOptions(); parallelOptions.MaxDegreeOfParallelism = processorLimit; }
public static Tuple <double, double> EstimateObjectives(float[,] xImage, float[,] residuals, float[,] psf, float[,] xExplore, float[,] xAccelerated, float lambda, float alpha, float[,] psfCut, float[,] bMap) { Tuple <double, double> output = null; var CONVKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); using (var residualsCalculator = new PaddedConvolver(CONVKernel, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)))) { var residualsExplore = new float[xImage.GetLength(0), xImage.GetLength(1)]; var residualsAccelerated = new float[xImage.GetLength(0), xImage.GetLength(1)]; for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residualsExplore[i, j] = xExplore[i, j] - xImage[i, j]; residualsAccelerated[i, j] = xAccelerated[i, j] - xImage[i, j]; } } residualsCalculator.ConvolveInPlace(residualsExplore); residualsCalculator.ConvolveInPlace(residualsAccelerated); for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residualsExplore[i, j] = residuals[i, j] - residualsExplore[i, j]; residualsAccelerated[i, j] = residuals[i, j] - residualsAccelerated[i, j]; } } var objectiveExplore = Residuals.CalcPenalty(residualsExplore) + ElasticNet.CalcPenalty(xExplore, lambda, alpha); var objectiveAccelerated = Residuals.CalcPenalty(residualsAccelerated) + ElasticNet.CalcPenalty(xAccelerated, lambda, alpha); /* * var CORRKernel = PSF.CalcPaddedFourierCorrelation(psfCut, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); * var bMapFull = Residuals.CalcBMap(residualsExplore, CORRKernel, new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); * FitsIO.Write(bMap, "bMapOriginal.fits"); * FitsIO.Write(bMapFull, "bMapSanity.fits"); * var diff = new float[bMap.GetLength(0), bMap.GetLength(1)]; * for (int i = 0; i < xImage.GetLength(0); i++) * for (int j = 0; j < xImage.GetLength(1); j++) * diff[i, j] = bMap[i, j] - bMapFull[i, j]; * FitsIO.Write(diff, "bmapSanityDiff.fits");*/ output = new Tuple <double, double>(objectiveExplore, objectiveAccelerated); } return(output); }
public void ResetLipschitzMap(float[,] psf) { var psf2Local = PSF.CalcPSFSquared(psf); var maxFull = Residuals.GetMax(psf2Local); MaxLipschitz = maxFull; aMap = PSF.CalcAMap(psf, patch); var maxCut = Residuals.GetMax(psf2); for (int i = 0; i < psf2.GetLength(0); i++) { for (int j = 0; j < psf2.GetLength(1); j++) { psf2[i, j] *= (maxFull / maxCut); } } }
public void ResetAMap(float[,] psf) { var psf2Local = PSF.CalcPSFSquared(psf); var maxFull = Residuals.GetMax(psf2Local); aMap = PSF.CalcAMap(psf, totalSize); this.psf = psf; var maxCut = Residuals.GetMax(psf2); for (int i = 0; i < psf2.GetLength(0); i++) { for (int j = 0; j < psf2.GetLength(1); j++) { psf2[i, j] *= (maxFull / maxCut); } } }
public static void GenerateSerialCDExample(string simulatedLocation, string outputFolder) { var data = MeasurementData.LoadSimulatedPoints(simulatedLocation); var cellSize = 1.0 / 3600.0 * Math.PI / 180.0; var c = new GriddingConstants(data.VisibilitiesCount, 256, 8, 4, 512, (float)cellSize, 1, 0.0); var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies); var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies); var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var corrKernel = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, c.GridSize, c.GridSize)); Directory.CreateDirectory(outputFolder); var reconstruction = new float[c.GridSize, c.GridSize]; var residualVis = data.Visibilities; var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var fastCD = new FastSerialCD(totalSize, psf); var lambda = 0.50f * fastCD.MaxLipschitz; var alpha = 0.2f; for (int cycle = 0; cycle < 100; cycle++) { var dirtyGrid = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies); var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); var gradients = Residuals.CalcGradientMap(dirtyImage, corrKernel, totalSize); Tools.WriteToMeltCSV(Common.PSF.Cut(reconstruction), Path.Combine(outputFolder, "model_CD_" + cycle + ".csv")); Tools.WriteToMeltCSV(gradients, Path.Combine(outputFolder, "gradients_CD_" + cycle + ".csv")); fastCD.Deconvolve(reconstruction, gradients, lambda, alpha, 4); FFT.Shift(reconstruction); var xGrid = FFT.Forward(reconstruction); FFT.Shift(reconstruction); var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies); residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags); } }
public void ISTAStep(float[,] xImage, float[,] residuals, float[,] psf, float lambda, float alpha) { var xOld = Copy(xImage); var corrKernel = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); var gradients = Residuals.CalcGradientMap(residuals, corrKernel, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var lipschitz = (float)PSF.CalcMaxLipschitz(psf) * xImage.Length; for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { var tmp = gradients[i, j] + xImage[i, j] * lipschitz; tmp = ElasticNet.ProximalOperator(tmp, lipschitz, lambda, alpha); xImage[i, j] = tmp; } } //update residuals for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xOld[i, j] = xImage[i, j] - xOld[i, j]; } } var convKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); var residualsCalculator = new PaddedConvolver(convKernel, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); residualsCalculator.ConvolveInPlace(xOld); for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residuals[i, j] -= xOld[i, j]; } } }
private static Tuple <double, double> CalcObjectives(float[,] xImage, float[,] residuals, float[,] psf, float[,] xExplore, float[,] xAccelerated, float lambda, float alpha) { Tuple <double, double> output = null; var CONVKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); using (var residualsCalculator = new PaddedConvolver(CONVKernel, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)))) { var residualsExplore = new float[xImage.GetLength(0), xImage.GetLength(1)]; var residualsAccelerated = new float[xImage.GetLength(0), xImage.GetLength(1)]; for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residualsExplore[i, j] = xExplore[i, j] - xImage[i, j]; residualsAccelerated[i, j] = xAccelerated[i, j] - xImage[i, j]; } } residualsCalculator.ConvolveInPlace(residualsExplore); residualsCalculator.ConvolveInPlace(residualsAccelerated); for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residualsExplore[i, j] -= residuals[i, j]; residualsAccelerated[i, j] -= residuals[i, j]; } } var objectiveExplore = Residuals.CalcPenalty(residualsExplore) + ElasticNet.CalcPenalty(xExplore, lambda, alpha); var objectiveAccelerated = Residuals.CalcPenalty(residualsAccelerated) + ElasticNet.CalcPenalty(xAccelerated, lambda, alpha); output = new Tuple <double, double>(objectiveExplore, objectiveAccelerated); } return(output); }
private static ReconstructionInfo ReconstructGradientApprox(Data input, float[,] fullPsf, string folder, int cutFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon) { var info = new ReconstructionInfo(); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var psfBMap = psfCut; var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))); var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))); var fastCD = new FastSerialCD(totalSize, psfCut); var fastCD2 = new FastSerialCD(totalSize, psfCut); fastCD2.ResetLipschitzMap(fullPsf); FitsIO.Write(psfCut, folder + cutFactor + "psf.fits"); var lambda = LAMBDA_GLOBAL * fastCD.MaxLipschitz; var lambdaTrue = (float)(LAMBDA_GLOBAL * PSF.CalcMaxLipschitz(fullPsf)); var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; DeconvolutionResult lastResult = null; var firstTimeConverged = false; var lastLambda = 0.0f; for (int cycle = 0; cycle < maxMajor; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits"); //calc data and reg penalty var dataPenalty = Residuals.CalcPenalty(dirtyImage); var regPenalty = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha); info.lastDataPenalty = dataPenalty; info.lastRegPenalty = regPenalty; var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + ".fits"); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = Math.Max(currentSideLobe / alpha, lambda); writer.Write(cycle + ";" + currentLambda + ";" + currentSideLobe + ";" + ";" + fastCD2.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";");; writer.Flush(); //check wether we can minimize the objective further with the current psf var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff; var minimumReached = (lastResult != null && lastResult.Converged && fastCD2.GetAbsMaxDiff(xImage, dirtyImage, lambdaTrue, alpha) < MAJOR_EPSILON && currentLambda == lambda); if (lambda == lastLambda & !firstTimeConverged) { firstTimeConverged = true; minimumReached = false; } if (!objectiveReached & !minimumReached) { //writer.Write(firstTimeConverged + ";"); //writer.Flush(); info.totalDeconv.Start(); if (!firstTimeConverged) { lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon); } else { bMap = bMapCalculator2.Convolve(dirtyImage); //FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + "_full.fits"); maxB = Residuals.GetMax(bMap); correctionFactor = Math.Max(maxB / (maxDirty * fastCD2.MaxLipschitz), 1.0f); currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = Math.Max(currentSideLobe / alpha, lambdaTrue); info.totalDeconv.Start(); lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon); info.totalDeconv.Stop(); } info.totalDeconv.Stop(); FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits"); writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n"); writer.Flush(); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } else { writer.Write(false + ";0;0\n"); writer.Flush(); break; } lastLambda = currentLambda; } bMapCalculator.Dispose(); bMapCalculator2.Dispose(); return(info); }
public bool DeconvolveApprox(float[,] xImage, float[,] residuals, float[,] psf, float lambda, float alpha, Random random, int blockSize, int threadCount, int maxIteration = 100, float epsilon = 1e-4f, bool coldStart = false) { var xExplore = Copy(xImage); var xCorrection = new float[xImage.GetLength(0), xImage.GetLength(1)]; //calculate gradients for each pixel var PSFCorr = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); var gExplore = Residuals.CalcGradientMap(residuals, PSFCorr, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var gCorrection = new float[residuals.GetLength(0), residuals.GetLength(1)]; var psf2 = PSF.CalcPSFSquared(psf); if (coldStart) { var rec = new Rectangle(0, 0, xImage.GetLength(0), xImage.GetLength(1)); var fastCD = new FastSerialCD(rec, rec, psf, psf2); fastCD.Deconvolve(xExplore, gExplore, lambda, alpha, xImage.GetLength(0)); } yBlockSize = blockSize; xBlockSize = blockSize; degreeOfSeperability = CountNonZero(psf); tau = threadCount; //number of processors var maxLipschitz = (float)PSF.CalcMaxLipschitz(psf); var activeSet = GetActiveSet(xExplore, gExplore, lambda, alpha, maxLipschitz); var theta = DeconvolveAccelerated(xExplore, xCorrection, gExplore, gCorrection, psf2, ref activeSet, maxLipschitz, lambda, alpha, random, maxIteration, epsilon); var theta0 = tau / (float)activeSet.Count; var tmpTheta = theta < 1.0f ? ((theta * theta) / (1.0f - theta)) : theta0; for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xCorrection[i, j] = tmpTheta * xCorrection[i, j] + xExplore[i, j]; } } var objectives = CalcObjectives(xImage, residuals, psf, xExplore, xCorrection, lambda, alpha); //decide whether we take the correction or explore version if (objectives.Item2 < objectives.Item1) { //correction has the lower objective than explore for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xImage[i, j] = xCorrection[i, j]; } } } else { for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xImage[i, j] = xExplore[i, j]; } } } return(objectives.Item2 < objectives.Item1); }
private static void ReconstructMinorCycle(MeasurementData input, GriddingConstants c, int cutFactor, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, bool useAccelerated = true, int blockSize = 1, int maxCycle = 6) { var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2); var random = new Random(123); var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.1f, searchPercent, false, useAccelerated); using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)))) using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)))) using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf)) { var currentBMapCalculator = bMapCalculator; var maxLipschitz = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * maxLipschitz); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf)); var alpha = ALPHA; ApproxFast.LAMBDA_TEST = lambdaTrue; ApproxFast.ALPHA_TEST = alpha; var switchedToOtherPsf = false; var writer = new StreamWriter(folder + "/" + file + "_lambda.txt"); var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt")); var xImage = new float[c.GridSize, c.GridSize]; var residualVis = input.Visibilities; for (int cycle = 0; cycle < maxCycle; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits"); var minLambda = 0.0f; var dirtyCopy = Copy(dirtyImage); var xCopy = Copy(xImage); var currentLambda = 0f; //var residualsConvolver = new PaddedConvolver(PSF.CalcPaddedFourierConvolution(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))); for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++) { FitsIO.Write(dirtyImage, folder + "/dirtyMinor_" + minorCycle + ".fits"); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = currentBMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); if (minorCycle == 0) { minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha); } if (currentLambda < minLambda) { currentLambda = minLambda; } writer.WriteLine(cycle + ";" + minorCycle + ";" + currentLambda + ";" + minLambda); writer.Flush(); approx.DeconvolveTest(data, cycle, minorCycle, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f); FitsIO.Write(xImage, folder + "/xImageMinor_" + minorCycle + ".fits"); if (currentLambda == lambda | currentLambda == minLambda) { break; } Console.WriteLine("resetting residuals!!"); //reset dirtyImage with full PSF var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)]; Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j]; } }); residualsConvolver.ConvolveInPlace(residualsUpdate); Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j]; } }); } if (currentLambda == lambda & !switchedToOtherPsf) { approx.ResetAMap(fullPsf); currentBMapCalculator = bMapCalculator2; lambda = lambdaTrue; switchedToOtherPsf = true; writer.WriteLine("switched"); writer.Flush(); } FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits"); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies); residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags); } writer.Close(); } }
private static void Reconstruct(Data input, int cutFactor, float[,] fullPsf, string folder, string file, int threads, int blockSize, bool accelerated, float randomPercent, float searchPercent) { var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); var random = new Random(123); var approx = new ApproxFast(totalSize, psfCut, threads, blockSize, randomPercent, searchPercent, false, true); var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut)); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf)); var alpha = ALPHA; ApproxFast.LAMBDA_TEST = lambdaTrue; ApproxFast.ALPHA_TEST = alpha; var switchedToOtherPsf = false; var writer = new StreamWriter(folder + "/" + file + "_lambda.txt"); var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt")); var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; for (int cycle = 0; cycle < 7; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits"); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); writer.WriteLine("cycle" + ";" + currentLambda); writer.Flush(); approx.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f); FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits"); if (currentLambda == lambda & !switchedToOtherPsf) { approx.ResetAMap(fullPsf); lambda = lambdaTrue; switchedToOtherPsf = true; writer.WriteLine("switched"); writer.Flush(); } FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } writer.Close(); }
/// <summary> /// Major cycle implementation for the Serial CD /// </summary> /// <param name="obsName"></param> /// <param name="data"></param> /// <param name="c"></param> /// <param name="useGPU"></param> /// <param name="psfCutFactor"></param> /// <param name="maxMajorCycle"></param> /// <param name="lambda"></param> /// <param name="alpha"></param> /// <param name="deconvIterations"></param> /// <param name="deconvEpsilon"></param> public static void ReconstructSerialCD(string obsName, MeasurementData data, GriddingConstants c, bool useGPU, int psfCutFactor, int maxMajorCycle, float lambda, float alpha, int deconvIterations, float deconvEpsilon) { var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies); var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length]; for (int i = 0; i < data.Visibilities.GetLength(0); i++) { for (int j = 0; j < data.Visibilities.GetLength(1); j++) { for (int k = 0; k < data.Visibilities.GetLength(2); k++) { if (!data.Flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var totalWatch = new Stopwatch(); var currentWatch = new Stopwatch(); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(psf, psfCutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(psf, psfCutFactor); IDeconvolver deconvolver = null; if (useGPU & GPUSerialCD.IsGPUSupported()) { deconvolver = new GPUSerialCD(totalSize, psfCut, 1000); } else if (useGPU & !GPUSerialCD.IsGPUSupported()) { Console.WriteLine("GPU not supported by library. Switching to CPU implementation"); deconvolver = new FastSerialCD(totalSize, psfCut); } else { deconvolver = new FastSerialCD(totalSize, psfCut); } var psfBMap = psfCut; using (var gCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1)))) using (var gCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)))) { var currentGCalculator = gCalculator; var maxLipschitz = PSF.CalcMaxLipschitz(psfCut); var lambdaLipschitz = (float)(lambda * maxLipschitz); var lambdaTrue = (float)(lambda * PSF.CalcMaxLipschitz(psf)); var switchedToOtherPsf = false; var xImage = new float[c.GridSize, c.GridSize]; var residualVis = data.Visibilities; DeconvolutionResult lastResult = null; for (int cycle = 0; cycle < maxMajorCycle; cycle++) { Console.WriteLine("Beginning Major cycle " + cycle); var dirtyGrid = IDG.GridW(c, metadata, residualVis, data.UVW, data.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, obsName + "_dirty_serial_majorCycle" + cycle + ".fits"); currentWatch.Restart(); totalWatch.Start(); var maxDirty = Residuals.GetMax(dirtyImage); var gradients = gCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(gradients); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambdaLipschitz); var objective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var absMax = deconvolver.GetAbsMaxDiff(xImage, gradients, lambdaTrue, alpha); if (absMax >= MAJOR_EPSILON) { lastResult = deconvolver.Deconvolve(xImage, gradients, currentLambda, alpha, deconvIterations, deconvEpsilon); } if (lambda == currentLambda & !switchedToOtherPsf) { currentGCalculator = gCalculator2; lambda = lambdaTrue; maxLipschitz = PSF.CalcMaxLipschitz(psf); switchedToOtherPsf = true; } FitsIO.Write(xImage, obsName + "_model_serial_majorCycle" + cycle + ".fits"); currentWatch.Stop(); totalWatch.Stop(); if (absMax < MAJOR_EPSILON) { break; } FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(c, metadata, xGrid, data.UVW, data.Frequencies); residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags); } Console.WriteLine("Reconstruction finished in (seconds): " + totalWatch.Elapsed.TotalSeconds); } }
public bool DeconvolveGreedy(float[,] xImage, float[,] residuals, float[,] psf, float lambda, float alpha, Random random, int blockSize, int threadCount, int maxIteration = 100, float epsilon = 1e-4f) { var xExplore = Copy(xImage); var xCorrection = new float[xImage.GetLength(0), xImage.GetLength(1)]; var PSFCorr = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); //calculate gradients for each pixel var gExplore = Residuals.CalcGradientMap(residuals, PSFCorr, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var gCorrection = new float[residuals.GetLength(0), residuals.GetLength(1)]; var psf2 = PSF.CalcPSFSquared(psf); FitsIO.Write(gExplore, "gExplore.fits"); yBlockSize = blockSize; xBlockSize = blockSize; degreeOfSeperability = CountNonZero(psf); tau = threadCount; //number of processors var maxLipschitz = (float)PSF.CalcMaxLipschitz(psf); var theta = Greedy(xExplore, xCorrection, gExplore, gCorrection, psf2, maxLipschitz, lambda, alpha, random, maxIteration, epsilon); //decide which version should be taken# var CONVKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, residuals.GetLength(0), residuals.GetLength(1))); var residualsCalculator = new PaddedConvolver(CONVKernel, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var theta0 = tau / (float)(xExplore.Length / (yBlockSize * xBlockSize)); var tmpTheta = theta < 1.0f ? ((theta * theta) / (1.0f - theta)) : theta0; //calculate residuals var residualsExplore = Copy(xExplore); var residualsAccelerated = Copy(xExplore); for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residualsExplore[i, j] -= xImage[i, j]; residualsAccelerated[i, j] += tmpTheta * xCorrection[i, j] - xImage[i, j]; xCorrection[i, j] = tmpTheta * xCorrection[i, j] + xExplore[i, j]; } } FitsIO.Write(xExplore, "xExplore.fits"); FitsIO.Write(xCorrection, "xAcc.fits"); residualsCalculator.ConvolveInPlace(residualsExplore); residualsCalculator.ConvolveInPlace(residualsAccelerated); for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { residualsExplore[i, j] -= residuals[i, j]; residualsAccelerated[i, j] -= residuals[i, j]; } } var objectiveExplore = Residuals.CalcPenalty(residualsExplore) + ElasticNet.CalcPenalty(xExplore, lambda, alpha); var objectiveAcc = Residuals.CalcPenalty(residualsAccelerated) + ElasticNet.CalcPenalty(xCorrection, lambda, alpha); if (objectiveAcc < objectiveExplore) { for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xImage[i, j] = xCorrection[i, j]; } } } else { for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xImage[i, j] = xExplore[i, j]; } } } return(objectiveAcc < objectiveExplore); }
private static void ReconstructPCDM(MeasurementData input, GriddingConstants c, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, int processorCount) { var totalWatch = new Stopwatch(); var currentWatch = new Stopwatch(); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(fullPsf, CUT_FACTOR_PCDM); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, CUT_FACTOR_PCDM); var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2); var random = new Random(123); var pcdm = new ParallelCoordinateDescent(totalSize, psfCut, 1, 1000, searchPercent); var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies); using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)))) using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)))) using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf)) { var currentBMapCalculator = bMapCalculator; var maxLipschitz = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * maxLipschitz); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf)); var alpha = ALPHA; var switchedToOtherPsf = false; var writer = new StreamWriter(folder + "/" + file + ".txt"); var xImage = new float[c.GridSize, c.GridSize]; var residualVis = input.Visibilities; ParallelCoordinateDescent.PCDMStatistics lastResult = null; for (int cycle = 0; cycle < 6; cycle++) { Console.WriteLine("Beginning Major cycle " + cycle); var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits"); currentWatch.Restart(); totalWatch.Start(); var breakMajor = false; var minLambda = 0.0f; var dirtyCopy = Copy(dirtyImage); var xCopy = Copy(xImage); var currentLambda = 0f; var currentObjective = 0.0; var absMax = 0.0f; for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++) { Console.WriteLine("Beginning Minor Cycle " + minorCycle); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = currentBMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); if (minorCycle == 0) { minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha); } if (currentLambda < minLambda) { currentLambda = minLambda; } currentObjective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); absMax = pcdm.GetAbsMax(xImage, bMap, lambdaTrue, alpha); if (absMax < MAJOR_STOP) { breakMajor = true; break; } lastResult = pcdm.Deconvolve(xImage, bMap, currentLambda, alpha, 100, 1e-5f); if (currentLambda == lambda | currentLambda == minLambda) { break; } var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)]; Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j]; } }); residualsConvolver.ConvolveInPlace(residualsUpdate); Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j]; } }); } currentWatch.Stop(); totalWatch.Stop(); writer.WriteLine(cycle + ";" + currentLambda + ";" + currentObjective + ";" + absMax + ";" + lastResult.IterationCount + ";" + totalWatch.Elapsed.TotalSeconds + ";" + currentWatch.Elapsed.TotalSeconds); writer.Flush(); FitsIO.Write(xImage, folder + "/xImage_pcdm_" + cycle + ".fits"); if (breakMajor) { break; } if (currentLambda == lambda & !switchedToOtherPsf) { pcdm.ResetAMap(fullPsf); currentBMapCalculator = bMapCalculator2; lambda = lambdaTrue; switchedToOtherPsf = true; writer.WriteLine("switched"); writer.Flush(); } FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies); residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags); } writer.Close(); } }
private static ReconstructionInfo Reconstruct(Data input, float fullLipschitz, float[,] maskedPsf, string folder, float maskFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon, bool maskPsf2) { var info = new ReconstructionInfo(); var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(maskedPsf, totalSize), new Rectangle(0, 0, maskedPsf.GetLength(0), maskedPsf.GetLength(1))); var maskedPsf2 = PSF.CalcPSFSquared(maskedPsf); if (maskPsf2) { Mask(maskedPsf2, 1e-5f); } writer.WriteLine((CountNonZero(maskedPsf2) - maskedPsf2.Length) / (double)maskedPsf2.Length); var fastCD = new FastSerialCD(totalSize, totalSize, maskedPsf, maskedPsf2); FitsIO.Write(maskedPsf, folder + maskFactor + "psf.fits"); var lambda = 0.4f * fastCD.MaxLipschitz; var lambdaTrue = 0.4f * fullLipschitz; var alpha = 0.1f; var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; DeconvolutionResult lastResult = null; for (int cycle = 0; cycle < maxMajor; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits"); //calc data and reg penalty var dataPenalty = Residuals.CalcPenalty(dirtyImage); var regPenalty = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha); info.lastDataPenalty = dataPenalty; info.lastRegPenalty = regPenalty; bMapCalculator.ConvolveInPlace(dirtyImage); //FitsIO.Write(dirtyImage, folder + dirtyPrefix + "bmap_" + cycle + ".fits"); var currentLambda = lambda; writer.Write(cycle + ";" + currentLambda + ";" + Residuals.GetMax(dirtyImage) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";"); writer.Flush(); //check wether we can minimize the objective further with the current psf var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff; var minimumReached = (lastResult != null && lastResult.IterationCount < 100 && lastResult.Converged); if (!objectiveReached & !minimumReached) { info.totalDeconv.Start(); lastResult = fastCD.Deconvolve(xImage, dirtyImage, currentLambda, alpha, 50000, epsilon); info.totalDeconv.Stop(); FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits"); writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n"); writer.Flush(); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } else { writer.Write(false + ";0;0"); writer.Flush(); break; } } return(info); }
public static void DebugSimulatedApprox() { var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits"); var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits"); var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm); var visibilitiesCount = visibilities.Length; int gridSize = 256; int subgridsize = 8; int kernelSize = 4; int max_nr_timesteps = 1024; double cellSize = 1.0 / 3600.0 * PI / 180.0; var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f); var watchTotal = new Stopwatch(); var watchForward = new Stopwatch(); var watchBackwards = new Stopwatch(); var watchDeconv = new Stopwatch(); watchTotal.Start(); var metadata = Partitioner.CreatePartition(c, uvw, frequencies); var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies); var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var psfCut = PSF.Cut(psf); FitsIO.Write(psfCut, "psfCut.fits"); var random = new Random(123); var totalSize = new Rectangle(0, 0, gridSize, gridSize); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); var fastCD = new FastSerialCD(totalSize, psfCut); //fastCD.ResetAMap(psf); var lambda = 0.5f * fastCD.MaxLipschitz; var alpha = 0.8f; var approx = new ApproxParallel(); var approx2 = new ApproxFast(totalSize, psfCut, 4, 8, 0f, 0.25f, false, true); var xImage = new float[gridSize, gridSize]; var residualVis = visibilities; /*var truth = new double[gridSize, gridSize]; * truth[30, 30] = 1.0; * truth[35, 36] = 1.5; * var truthVis = IDG.ToVisibilities(c, metadata, truth, uvw, frequencies); * visibilities = truthVis; * var residualVis = truthVis;*/ var data = new ApproxFast.TestingData(new StreamWriter("approxConvergence.txt")); for (int cycle = 0; cycle < 4; cycle++) { //FORWARD watchForward.Start(); var dirtyGrid = IDG.Grid(c, metadata, residualVis, uvw, frequencies); var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, "dirty_" + cycle + ".fits"); watchForward.Stop(); //DECONVOLVE watchDeconv.Start(); //approx.ISTAStep(xImage, dirtyImage, psf, lambda, alpha); //FitsIO.Write(xImage, "xIsta.fits"); //FitsIO.Write(dirtyImage, "dirtyFista.fits"); //bMapCalculator.ConvolveInPlace(dirtyImage); //FitsIO.Write(dirtyImage, "bMap_" + cycle + ".fits"); //var result = fastCD.Deconvolve(xImage, dirtyImage, 0.5f * fastCD.MaxLipschitz, 0.8f, 1000, 1e-4f); //var converged = approx.DeconvolveActiveSet(xImage, dirtyImage, psfCut, lambda, alpha, random, 8, 1, 1); //var converged = approx.DeconvolveGreedy(xImage, dirtyImage, psfCut, lambda, alpha, random, 4, 4, 500); //var converged = approx.DeconvolveApprox(xImage, dirtyImage, psfCut, lambda, alpha, random, 1, threads, 500, 1e-4f, cycle == 0); approx2.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, psf, lambda, alpha, random, 10, 1e-4f); if (data.converged) { Console.WriteLine("-----------------------------CONVERGED!!!!------------------------"); } else { Console.WriteLine("-------------------------------not converged----------------------"); } FitsIO.Write(xImage, "xImageApprox_" + cycle + ".fits"); watchDeconv.Stop(); //BACKWARDS watchBackwards.Start(); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies); residualVis = Visibilities.Substract(visibilities, modelVis, flags); watchBackwards.Stop(); } var dirtyGridCheck = IDG.Grid(c, metadata, residualVis, uvw, frequencies); var dirtyCheck = FFT.Backward(dirtyGridCheck, c.VisibilitiesCount); FFT.Shift(dirtyCheck); var l2Penalty = Residuals.CalcPenalty(ToFloatImage(dirtyCheck)); var elasticPenalty = ElasticNet.CalcPenalty(xImage, (float)lambda, (float)alpha); var sum = l2Penalty + elasticPenalty; data.writer.Close(); }
public static void GeneratePSFs(string simulatedLocation, string outputFolder) { var data = MeasurementData.LoadSimulatedPoints(simulatedLocation); var c = MeasurementData.CreateSimulatedStandardParams(data.VisibilitiesCount); var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies); var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies); var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); Directory.CreateDirectory(outputFolder); var maskedPsf = Copy(psf); Tools.Mask(maskedPsf, 2); var reverseMasked = Copy(psf); Tools.ReverseMask(reverseMasked, 2); var psf2 = PSF.CalcPSFSquared(psf); var psf2Cut = PSF.CalcPSFSquared(maskedPsf); Tools.WriteToMeltCSV(psf, Path.Combine(outputFolder, "psf.csv")); Tools.WriteToMeltCSV(maskedPsf, Path.Combine(outputFolder, "psfCut.csv")); Tools.WriteToMeltCSV(reverseMasked, Path.Combine(outputFolder, "psfReverseCut.csv")); Tools.WriteToMeltCSV(psf2, Path.Combine(outputFolder, "psfSquared.csv")); Tools.WriteToMeltCSV(psf2Cut, Path.Combine(outputFolder, "psfSquaredCut.csv")); var x = new float[c.GridSize, c.GridSize]; x[10, 10] = 1.0f; var convKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, c.GridSize, c.GridSize)); var corrKernel = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, c.GridSize, c.GridSize)); using (var convolver = new PaddedConvolver(convKernel, new Rectangle(0, 0, c.GridSize, c.GridSize))) using (var correlator = new PaddedConvolver(corrKernel, new Rectangle(0, 0, c.GridSize, c.GridSize))) { var zeroPadded = convolver.Convolve(x); var psf2Edge = correlator.Convolve(zeroPadded); Tools.WriteToMeltCSV(zeroPadded, Path.Combine(outputFolder, "psfZeroPadding.csv")); Tools.WriteToMeltCSV(psf2Edge, Path.Combine(outputFolder, "psfSquaredEdge.csv")); } convKernel = PSF.CalcPaddedFourierConvolution(psf, new Rectangle(0, 0, 0, 0)); using (var convolver = new PaddedConvolver(convKernel, new Rectangle(0, 0, 0, 0))) Tools.WriteToMeltCSV(convolver.Convolve(x), Path.Combine(outputFolder, "psfCircular.csv")); //================================================= Reconstruct ============================================================= var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var reconstruction = new float[c.GridSize, c.GridSize]; var fastCD = new FastSerialCD(totalSize, psf); var lambda = 0.50f * fastCD.MaxLipschitz; var alpha = 0.2f; var residualVis = data.Visibilities; for (int cycle = 0; cycle < 5; cycle++) { Console.WriteLine("in cycle " + cycle); var dirtyGrid = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies); var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); var gradients = Residuals.CalcGradientMap(dirtyImage, corrKernel, totalSize); if (cycle == 0) { Tools.WriteToMeltCSV(dirtyImage, Path.Combine(outputFolder, "dirty.csv")); Tools.WriteToMeltCSV(gradients, Path.Combine(outputFolder, "gradients.csv")); } fastCD.Deconvolve(reconstruction, gradients, lambda, alpha, 10000, 1e-5f); FFT.Shift(reconstruction); var xGrid = FFT.Forward(reconstruction); FFT.Shift(reconstruction); var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies); residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags); } //FitsIO.Write(reconstruction, Path.Combine(outputFolder,"xImage.fits")); Tools.WriteToMeltCSV(reconstruction, Path.Combine(outputFolder, "elasticNet.csv")); }
private static void ReconstructSerial(MeasurementData input, GriddingConstants c, float[,] fullPsf, string folder, string file, int processorCount) { var totalWatch = new Stopwatch(); var currentWatch = new Stopwatch(); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(fullPsf, CUT_FACTOR_SERIAL); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, CUT_FACTOR_SERIAL); var fastCD = new FastSerialCD(totalSize, psfCut, processorCount); var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies); var writer = new StreamWriter(folder + "/" + file + ".txt"); var psfBMap = psfCut; using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1)))) using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)))) { var currentBMapCalculator = bMapCalculator; var maxLipschitz = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * maxLipschitz); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf)); var alpha = ALPHA; var switchedToOtherPsf = false; var xImage = new float[c.GridSize, c.GridSize]; var residualVis = input.Visibilities; DeconvolutionResult lastResult = null; for (int cycle = 0; cycle < 6; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits"); currentWatch.Restart(); totalWatch.Start(); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = Math.Max(currentSideLobe / alpha, lambda); var objective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var absMax = fastCD.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha); if (absMax >= MAJOR_STOP) { lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, 1e-5f); } if (lambda == currentLambda & !switchedToOtherPsf) { currentBMapCalculator = bMapCalculator2; lambda = lambdaTrue; switchedToOtherPsf = true; } currentWatch.Stop(); totalWatch.Stop(); writer.WriteLine(cycle + ";" + currentLambda + ";" + objective + ";" + absMax + ";" + lastResult.IterationCount + ";" + totalWatch.Elapsed.TotalSeconds + ";" + currentWatch.Elapsed.TotalSeconds); writer.Flush(); if (absMax < MAJOR_STOP) { break; } FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies); residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags); } } }
/// <summary> /// Major cycle implemnentation for the parallel coordinate descent algorithm /// </summary> /// <param name="data"></param> /// <param name="c"></param> /// <param name="psfCutFactor"></param> /// <param name="maxMajorCycle"></param> /// <param name="maxMinorCycle"></param> /// <param name="lambda"></param> /// <param name="alpha"></param> /// <param name="deconvIterations"></param> /// <param name="deconvEpsilon"></param> public static void ReconstructPCDM(string obsName, MeasurementData data, GriddingConstants c, int psfCutFactor, int maxMajorCycle, int maxMinorCycle, float lambda, float alpha, int deconvIterations, float deconvEpsilon) { var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies); var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length]; for (int i = 0; i < data.Visibilities.GetLength(0); i++) { for (int j = 0; j < data.Visibilities.GetLength(1); j++) { for (int k = 0; k < data.Visibilities.GetLength(2); k++) { if (!data.Flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.Grid(c, metadata, psfVis, data.UVW, data.Frequencies); var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var totalWatch = new Stopwatch(); var currentWatch = new Stopwatch(); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(psf, psfCutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(psf, psfCutFactor); var sidelobeHalf = PSF.CalcMaxSidelobe(psf, 2); var pcdm = new ParallelCoordinateDescent(totalSize, psfCut, Environment.ProcessorCount, 1000); using (var gCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)))) using (var gCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)))) using (var residualsConvolver = new PaddedConvolver(totalSize, psf)) { var currentGCalculator = gCalculator; var maxLipschitz = PSF.CalcMaxLipschitz(psfCut); var lambdaLipschitz = (float)(lambda * maxLipschitz); var lambdaTrue = (float)(lambda * PSF.CalcMaxLipschitz(psf)); var switchedToOtherPsf = false; var xImage = new float[c.GridSize, c.GridSize]; var residualVis = data.Visibilities; ParallelCoordinateDescent.PCDMStatistics lastResult = null; for (int cycle = 0; cycle < maxMajorCycle; cycle++) { Console.WriteLine("Beginning Major cycle " + cycle); var dirtyGrid = IDG.GridW(c, metadata, residualVis, data.UVW, data.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, obsName + "_dirty_pcdm_majorCycle" + cycle + ".fits"); currentWatch.Restart(); totalWatch.Start(); var breakMajor = false; var minLambda = 0.0f; var dirtyCopy = Copy(dirtyImage); var xCopy = Copy(xImage); var currentLambda = 0f; var currentObjective = 0.0; var absMax = 0.0f; for (int minorCycle = 0; minorCycle < maxMinorCycle; minorCycle++) { Console.WriteLine("Beginning Minor Cycle " + minorCycle); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = currentGCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = (float)Math.Max(currentSideLobe / alpha, lambdaLipschitz); if (minorCycle == 0) { minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha); } if (currentLambda < minLambda) { currentLambda = minLambda; } currentObjective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); absMax = pcdm.GetAbsMax(xImage, bMap, lambdaTrue, alpha); if (absMax < MAJOR_EPSILON) { breakMajor = true; break; } lastResult = pcdm.Deconvolve(xImage, bMap, currentLambda, alpha, 40, deconvEpsilon); if (currentLambda == lambda | currentLambda == minLambda) { break; } var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)]; Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j]; } }); residualsConvolver.ConvolveInPlace(residualsUpdate); Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j]; } }); } currentWatch.Stop(); totalWatch.Stop(); if (breakMajor) { break; } if (currentLambda == lambda & !switchedToOtherPsf) { pcdm.ResetAMap(psf); currentGCalculator = gCalculator2; lambda = lambdaTrue; switchedToOtherPsf = true; } FitsIO.Write(xImage, obsName + "_model_pcdm_majorCycle" + cycle + ".fits"); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(c, metadata, xGrid, data.UVW, data.Frequencies); residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags); } Console.WriteLine("Reconstruction finished in (seconds): " + totalWatch.Elapsed.TotalSeconds); } }