private static void ReconstructRandom(MeasurementData input, GriddingConstants c, float[,] psf, int blockSize, int iterCount, string file)
{
var cutFactor = 8;
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(psf, cutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(psf, cutFactor);
var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut));
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(psf));
var alpha = ALPHA;
ApproxFast.LAMBDA_TEST = lambdaTrue;
ApproxFast.ALPHA_TEST = alpha;
var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);
var random = new Random(123);
var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.0f, 0.0f, false, true, false);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
var data = new ApproxFast.TestingData(new StreamWriter(file + "_tmp.txt"));
var xImage = new float[c.GridSize, c.GridSize];
var xCorr = Copy(xImage);
var residualVis = input.Visibilities;
var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = bMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);
var gCorr = new float[c.GridSize, c.GridSize];
var shared = new ApproxFast.SharedData(currentLambda, alpha, 1, 1, 8, CountNonZero(psfCut), approx.psf2, approx.aMap, xImage, xCorr, bMap, gCorr, new Random());
shared.ActiveSet = ApproxFast.GetActiveSet(xImage, bMap, shared.YBlockSize, shared.XBlockSize, lambda, alpha, shared.AMap);
shared.BlockLock = new int[shared.ActiveSet.Count];
shared.maxLipschitz = (float)PSF.CalcMaxLipschitz(psfCut);
shared.MaxConcurrentIterations = 1000;
approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, 1, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf);
var output = Tools.LMC.CutN132Remnant(xImage);
Tools.WriteToMeltCSV(output.Item1, file + "_1k.csv", output.Item2, output.Item3);
FitsIO.Write(output.Item1, file + "_1k.fits");
FitsIO.Write(xImage, file + "_1k2.fits");
approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, iterCount, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf);
output = Tools.LMC.CutN132Remnant(xImage);
Tools.WriteToMeltCSV(output.Item1, file + "_10k.csv", output.Item2, output.Item3);
FitsIO.Write(output.Item1, file + "_10k.fits");
FitsIO.Write(xImage, file + "_10k2.fits");
}
private static void ReconstructMinorCycle(MeasurementData input, GriddingConstants c, int cutFactor, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, bool useAccelerated = true, int blockSize = 1, int maxCycle = 6)
{
var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(fullPsf, cutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor);
var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2);
var random = new Random(123);
var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.1f, searchPercent, false, useAccelerated);
using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf))
{
var currentBMapCalculator = bMapCalculator;
var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * maxLipschitz);
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
var alpha = ALPHA;
ApproxFast.LAMBDA_TEST = lambdaTrue;
ApproxFast.ALPHA_TEST = alpha;
var switchedToOtherPsf = false;
var writer = new StreamWriter(folder + "/" + file + "_lambda.txt");
var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt"));
var xImage = new float[c.GridSize, c.GridSize];
var residualVis = input.Visibilities;
for (int cycle = 0; cycle < maxCycle; cycle++)
{
Console.WriteLine("cycle " + cycle);
var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");
var minLambda = 0.0f;
var dirtyCopy = Copy(dirtyImage);
var xCopy = Copy(xImage);
var currentLambda = 0f;
//var residualsConvolver = new PaddedConvolver(PSF.CalcPaddedFourierConvolution(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)));
for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++)
{
FitsIO.Write(dirtyImage, folder + "/dirtyMinor_" + minorCycle + ".fits");
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = currentBMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);
if (minorCycle == 0)
{
minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
}
if (currentLambda < minLambda)
{
currentLambda = minLambda;
}
writer.WriteLine(cycle + ";" + minorCycle + ";" + currentLambda + ";" + minLambda);
writer.Flush();
approx.DeconvolveTest(data, cycle, minorCycle, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f);
FitsIO.Write(xImage, folder + "/xImageMinor_" + minorCycle + ".fits");
if (currentLambda == lambda | currentLambda == minLambda)
{
break;
}
Console.WriteLine("resetting residuals!!");
//reset dirtyImage with full PSF
var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
}
});
residualsConvolver.ConvolveInPlace(residualsUpdate);
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
}
});
}
if (currentLambda == lambda & !switchedToOtherPsf)
{
approx.ResetAMap(fullPsf);
currentBMapCalculator = bMapCalculator2;
lambda = lambdaTrue;
switchedToOtherPsf = true;
writer.WriteLine("switched");
writer.Flush();
}
FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits");
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
}
writer.Close();
}
}
private static void Reconstruct(Data input, int cutFactor, float[,] fullPsf, string folder, string file, int threads, int blockSize, bool accelerated, float randomPercent, float searchPercent)
{
var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize);
var psfCut = PSF.Cut(fullPsf, cutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
var random = new Random(123);
var approx = new ApproxFast(totalSize, psfCut, threads, blockSize, randomPercent, searchPercent, false, true);
var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut));
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
var alpha = ALPHA;
ApproxFast.LAMBDA_TEST = lambdaTrue;
ApproxFast.ALPHA_TEST = alpha;
var switchedToOtherPsf = false;
var writer = new StreamWriter(folder + "/" + file + "_lambda.txt");
var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt"));
var xImage = new float[input.c.GridSize, input.c.GridSize];
var residualVis = input.visibilities;
for (int cycle = 0; cycle < 7; cycle++)
{
Console.WriteLine("cycle " + cycle);
var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = bMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);
writer.WriteLine("cycle" + ";" + currentLambda);
writer.Flush();
approx.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f);
FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits");
if (currentLambda == lambda & !switchedToOtherPsf)
{
approx.ResetAMap(fullPsf);
lambda = lambdaTrue;
switchedToOtherPsf = true;
writer.WriteLine("switched");
writer.Flush();
}
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies);
residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags);
}
writer.Close();
}
public static void DebugSimulatedApprox()
{
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits");
var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
var visibilitiesCount = visibilities.Length;
int gridSize = 256;
int subgridsize = 8;
int kernelSize = 4;
int max_nr_timesteps = 1024;
double cellSize = 1.0 / 3600.0 * PI / 180.0;
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
var watchTotal = new Stopwatch();
var watchForward = new Stopwatch();
var watchBackwards = new Stopwatch();
var watchDeconv = new Stopwatch();
watchTotal.Start();
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var psfCut = PSF.Cut(psf);
FitsIO.Write(psfCut, "psfCut.fits");
var random = new Random(123);
var totalSize = new Rectangle(0, 0, gridSize, gridSize);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
var fastCD = new FastSerialCD(totalSize, psfCut);
//fastCD.ResetAMap(psf);
var lambda = 0.5f * fastCD.MaxLipschitz;
var alpha = 0.8f;
var approx = new ApproxParallel();
var approx2 = new ApproxFast(totalSize, psfCut, 4, 8, 0f, 0.25f, false, true);
var xImage = new float[gridSize, gridSize];
var residualVis = visibilities;
/*var truth = new double[gridSize, gridSize];
* truth[30, 30] = 1.0;
* truth[35, 36] = 1.5;
* var truthVis = IDG.ToVisibilities(c, metadata, truth, uvw, frequencies);
* visibilities = truthVis;
* var residualVis = truthVis;*/
var data = new ApproxFast.TestingData(new StreamWriter("approxConvergence.txt"));
for (int cycle = 0; cycle < 4; cycle++)
{
//FORWARD
watchForward.Start();
var dirtyGrid = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, "dirty_" + cycle + ".fits");
watchForward.Stop();
//DECONVOLVE
watchDeconv.Start();
//approx.ISTAStep(xImage, dirtyImage, psf, lambda, alpha);
//FitsIO.Write(xImage, "xIsta.fits");
//FitsIO.Write(dirtyImage, "dirtyFista.fits");
//bMapCalculator.ConvolveInPlace(dirtyImage);
//FitsIO.Write(dirtyImage, "bMap_" + cycle + ".fits");
//var result = fastCD.Deconvolve(xImage, dirtyImage, 0.5f * fastCD.MaxLipschitz, 0.8f, 1000, 1e-4f);
//var converged = approx.DeconvolveActiveSet(xImage, dirtyImage, psfCut, lambda, alpha, random, 8, 1, 1);
//var converged = approx.DeconvolveGreedy(xImage, dirtyImage, psfCut, lambda, alpha, random, 4, 4, 500);
//var converged = approx.DeconvolveApprox(xImage, dirtyImage, psfCut, lambda, alpha, random, 1, threads, 500, 1e-4f, cycle == 0);
approx2.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, psf, lambda, alpha, random, 10, 1e-4f);
if (data.converged)
{
Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
}
else
{
Console.WriteLine("-------------------------------not converged----------------------");
}
FitsIO.Write(xImage, "xImageApprox_" + cycle + ".fits");
watchDeconv.Stop();
//BACKWARDS
watchBackwards.Start();
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
residualVis = Visibilities.Substract(visibilities, modelVis, flags);
watchBackwards.Stop();
}
var dirtyGridCheck = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
var dirtyCheck = FFT.Backward(dirtyGridCheck, c.VisibilitiesCount);
FFT.Shift(dirtyCheck);
var l2Penalty = Residuals.CalcPenalty(ToFloatImage(dirtyCheck));
var elasticPenalty = ElasticNet.CalcPenalty(xImage, (float)lambda, (float)alpha);
var sum = l2Penalty + elasticPenalty;
data.writer.Close();
}
