public static void StartLMCReconstruction()
{
var folder = Path.Combine(P9_DATA_FOLDER, "large/fits/meerkat_tiny/");
var data = MeasurementData.LoadLMC(folder);
int gridSize = 3072;
int subgridsize = 32;
int kernelSize = 16;
int max_nr_timesteps = 1024;
double cellSize = 1.5 / 3600.0 * Math.PI / 180.0;
int wLayerCount = 24;
var maxW = 0.0;
for (int i = 0; i < data.UVW.GetLength(0); i++)
{
for (int j = 0; j < data.UVW.GetLength(1); j++)
{
maxW = Math.Max(maxW, Math.Abs(data.UVW[i, j, 2]));
}
}
maxW = Partitioner.MetersToLambda(maxW, data.Frequencies[data.Frequencies.Length - 1]);
double wStep = maxW / (wLayerCount);
var griddingConstants = new GriddingConstants(data.VisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep);
var obsName = "LMC";
var lambda = 1.0f;
var alpha = 0.01f;
var epsilon = 1e-5f;
var maxMajorCycles = 5;
MajorCycle.ReconstructSerialCD(obsName, data, griddingConstants, true, 16, maxMajorCycles, lambda, alpha, 30000, epsilon);
MajorCycle.ReconstructPCDM(obsName, data, griddingConstants, 32, maxMajorCycles, 3, lambda, alpha, 30, epsilon);
}
private static DirtyImage ForwardCalculateB(Intracommunicator comm, GriddingConstants c, List <List <Subgrid> > metadata, Complex[,,] visibilities, double[,,] uvw, double[] frequencies, Complex[,] PsfCorrelation, float[,] psfCut, float maxSidelobe, Stopwatch watchIdg)
{
Stopwatch another = new Stopwatch();
comm.Barrier();
if (comm.Rank == 0)
{
watchIdg.Start();
}
var localGrid = IDG.Grid(c, metadata, visibilities, uvw, frequencies);
float[,] image = null;
float maxSideLobeLevel = 0.0f;
var grid_total = comm.Reduce <Complex[, ]>(localGrid, SequentialSum, 0);
if (comm.Rank == 0)
{
var dirtyImage = FFT.BackwardFloat(grid_total, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
if (comm.Rank == 0)
{
FitsIO.Write(dirtyImage, "dirtyImage.fits");
}
maxSideLobeLevel = maxSidelobe * Residuals.GetMax(dirtyImage);
//remove spheroidal
image = Residuals.CalcGradientMap(dirtyImage, PsfCorrelation, new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
watchIdg.Stop();
}
comm.Broadcast(ref maxSideLobeLevel, 0);
comm.Broadcast(ref image, 0);
return(new DirtyImage(image, maxSideLobeLevel));
}
private static void GenerateApproxRandomProblem(string outputFolder)
{
var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\";
Console.WriteLine("Generating approx random images");
var data = MeasurementData.LoadLMC(folder);
int gridSize = 3072;
int subgridsize = 32;
int kernelSize = 16;
int max_nr_timesteps = 1024;
double cellSize = 1.5 / 3600.0 * PI / 180.0;
int wLayerCount = 24;
var maxW = 0.0;
for (int i = 0; i < data.UVW.GetLength(0); i++)
{
for (int j = 0; j < data.UVW.GetLength(1); j++)
{
maxW = Math.Max(maxW, Math.Abs(data.UVW[i, j, 2]));
}
}
maxW = Partitioner.MetersToLambda(maxW, data.Frequencies[data.Frequencies.Length - 1]);
double wStep = maxW / (wLayerCount);
var c = new GriddingConstants(data.VisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep);
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];
for (int i = 0; i < data.Visibilities.GetLength(0); i++)
{
for (int j = 0; j < data.Visibilities.GetLength(1); j++)
{
for (int k = 0; k < data.Visibilities.GetLength(2); k++)
{
if (!data.Flags[i, j, k])
{
psfVis[i, j, k] = new Complex(1.0, 0);
}
else
{
psfVis[i, j, k] = new Complex(0, 0);
}
}
}
}
Console.WriteLine("gridding psf");
var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies);
var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
Directory.CreateDirectory(outputFolder);
ReconstructRandom(data, c, psf, 1, 200, outputFolder + "/random__block1");
ReconstructRandom(data, c, psf, 8, 50, outputFolder + "/random_10k_block8");
}
private static void RunNotAccelerated(MeasurementData input, GriddingConstants c, float[,] fullPsf, string outFolder)
{
var file = "TestNotAccelerated";
var currentFolder = Path.Combine(outFolder, "NotAccelerated");
Directory.CreateDirectory(currentFolder);
ReconstructMinorCycle(input, c, 32, fullPsf, currentFolder, file, 3, 0.1f, false);
}
public static void Run()
{
var folder = @"C:\dev\GitHub\p9-data\small\fits\simulation_point\";
var data = DataLoading.SimulatedPoints.Load(folder);
var gridSizes = new int[] { 256, 512, 1024, 2048, 4096 };
Directory.CreateDirectory("GPUSpeedup");
var writer = new StreamWriter("GPUSpeedup/GPUSpeedup.txt", false);
writer.WriteLine("imgSize;iterCPU;timeCPU;iterGPU;timeGPU");
foreach (var gridSize in gridSizes)
{
var visibilitiesCount = data.visibilitiesCount;
int subgridsize = 8;
int kernelSize = 4;
int max_nr_timesteps = 1024;
double cellSize = (1.0 * 256 / gridSize) / 3600.0 * Math.PI / 180.0;
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
var metadata = Partitioner.CreatePartition(c, data.uvw, data.frequencies);
var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits"));
var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw.fits"));
var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length];
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var residualVis = data.visibilities;
var dirtyGrid = IDG.Grid(c, metadata, residualVis, data.uvw, data.frequencies);
var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
var totalSize = new Rectangle(0, 0, gridSize, gridSize);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)));
var bMapCPU = bMapCalculator.Convolve(dirtyImage);
var bMapGPU = bMapCalculator.Convolve(dirtyImage);
var fastCD = new FastSerialCD(totalSize, psf);
var gpuCD = new GPUSerialCD(totalSize, psf, 1000);
var lambda = 0.5f * fastCD.MaxLipschitz;
var alpha = 0.5f;
var xCPU = new float[gridSize, gridSize];
var cpuResult = fastCD.Deconvolve(xCPU, bMapCPU, lambda, alpha, 10000, 1e-8f);
FitsIO.Write(xCPU, "GPUSpeedup/cpuResult" + gridSize + ".fits");
var xGPU = new float[gridSize, gridSize];
var gpuResult = gpuCD.Deconvolve(xGPU, bMapGPU, lambda, alpha, 10000, 1e-8f);
FitsIO.Write(xCPU, "GPUSpeedup/gpuResult" + gridSize + ".fits");
writer.WriteLine(gridSize + ";" + cpuResult.IterationCount + ";" + cpuResult.ElapsedTime.TotalSeconds + ";" + gpuResult.IterationCount + ";" + gpuResult.ElapsedTime.TotalSeconds);
writer.Flush();
}
writer.Close();
}
private static void ReconstructRandom(MeasurementData input, GriddingConstants c, float[,] psf, int blockSize, int iterCount, string file)
{
var cutFactor = 8;
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(psf, cutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(psf, cutFactor);
var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut));
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(psf));
var alpha = ALPHA;
ApproxFast.LAMBDA_TEST = lambdaTrue;
ApproxFast.ALPHA_TEST = alpha;
var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);
var random = new Random(123);
var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.0f, 0.0f, false, true, false);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
var data = new ApproxFast.TestingData(new StreamWriter(file + "_tmp.txt"));
var xImage = new float[c.GridSize, c.GridSize];
var xCorr = Copy(xImage);
var residualVis = input.Visibilities;
var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = bMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);
var gCorr = new float[c.GridSize, c.GridSize];
var shared = new ApproxFast.SharedData(currentLambda, alpha, 1, 1, 8, CountNonZero(psfCut), approx.psf2, approx.aMap, xImage, xCorr, bMap, gCorr, new Random());
shared.ActiveSet = ApproxFast.GetActiveSet(xImage, bMap, shared.YBlockSize, shared.XBlockSize, lambda, alpha, shared.AMap);
shared.BlockLock = new int[shared.ActiveSet.Count];
shared.maxLipschitz = (float)PSF.CalcMaxLipschitz(psfCut);
shared.MaxConcurrentIterations = 1000;
approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, 1, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf);
var output = Tools.LMC.CutN132Remnant(xImage);
Tools.WriteToMeltCSV(output.Item1, file + "_1k.csv", output.Item2, output.Item3);
FitsIO.Write(output.Item1, file + "_1k.fits");
FitsIO.Write(xImage, file + "_1k2.fits");
approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, iterCount, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf);
output = Tools.LMC.CutN132Remnant(xImage);
Tools.WriteToMeltCSV(output.Item1, file + "_10k.csv", output.Item2, output.Item3);
FitsIO.Write(output.Item1, file + "_10k.fits");
FitsIO.Write(xImage, file + "_10k2.fits");
}
private static void RunBlocksize(MeasurementData input, GriddingConstants c, float[,] fullPsf, string outFolder)
{
var blockTest = new int[] { 2, 4, 8 };
foreach (var block in blockTest)
{
var file = "block" + block;
var currentFolder = Path.Combine(outFolder, "BlockSize");
Directory.CreateDirectory(currentFolder);
ReconstructMinorCycle(input, c, 32, fullPsf, currentFolder, file, 3, 0.1f, true, block);
}
}
private static void RunSearchPercent(MeasurementData input, GriddingConstants c, float[,] fullPsf, string outFolder)
{
var searchPercent = new float[] { 0.01f, 0.05f, 0.1f, 0.2f, 0.4f, 0.6f, 0.8f };
foreach (var percent in searchPercent)
{
var file = "SearchPercent" + percent;
var currentFolder = Path.Combine(outFolder, "Search");
Directory.CreateDirectory(currentFolder);
ReconstructMinorCycle(input, c, 32, fullPsf, currentFolder, file, 3, percent, false);
}
}
private static void RunPsfSize(MeasurementData input, GriddingConstants c, float[,] fullPsf, string outFolder)
{
var psfTest = new int[] { 4, 8, 16, 32, 64 };
foreach (var psfSize in psfTest)
{
var file = "PsfSize" + psfSize;
var currentFolder = Path.Combine(outFolder, "PsfSize");
Directory.CreateDirectory(currentFolder);
ReconstructMinorCycle(input, c, psfSize, fullPsf, currentFolder, file, 3, 0.1f, false);
}
}
private static float[,] CalculatePSF(Intracommunicator comm, GriddingConstants c, List <List <Subgrid> > metadata, double[,,] uvw, bool[,,] flags, double[] frequencies)
{
float[,] psf = null;
var localGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
var psf_total = comm.Reduce(localGrid, SequentialSum, 0);
if (comm.Rank == 0)
{
psf = FFT.BackwardFloat(psf_total, c.VisibilitiesCount);
FFT.Shift(psf);
}
comm.Broadcast(ref psf, 0);
return(psf);
}
public static void GenerateSerialCDExample(string simulatedLocation, string outputFolder)
{
var data = MeasurementData.LoadSimulatedPoints(simulatedLocation);
var cellSize = 1.0 / 3600.0 * Math.PI / 180.0;
var c = new GriddingConstants(data.VisibilitiesCount, 256, 8, 4, 512, (float)cellSize, 1, 0.0);
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies);
var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var corrKernel = PSF.CalcPaddedFourierCorrelation(psf, new Rectangle(0, 0, c.GridSize, c.GridSize));
Directory.CreateDirectory(outputFolder);
var reconstruction = new float[c.GridSize, c.GridSize];
var residualVis = data.Visibilities;
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var fastCD = new FastSerialCD(totalSize, psf);
var lambda = 0.50f * fastCD.MaxLipschitz;
var alpha = 0.2f;
for (int cycle = 0; cycle < 100; cycle++)
{
var dirtyGrid = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies);
var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
var gradients = Residuals.CalcGradientMap(dirtyImage, corrKernel, totalSize);
Tools.WriteToMeltCSV(Common.PSF.Cut(reconstruction), Path.Combine(outputFolder, "model_CD_" + cycle + ".csv"));
Tools.WriteToMeltCSV(gradients, Path.Combine(outputFolder, "gradients_CD_" + cycle + ".csv"));
fastCD.Deconvolve(reconstruction, gradients, lambda, alpha, 4);
FFT.Shift(reconstruction);
var xGrid = FFT.Forward(reconstruction);
FFT.Shift(reconstruction);
var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies);
residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
}
}
public static void StartSimulatedReconstruction()
{
var folder = Path.Combine(P9_DATA_FOLDER, "small/fits/simulation_point/");
var data = MeasurementData.LoadSimulatedPoints(folder);
int gridSize = 256;
int subgridsize = 8;
int kernelSize = 4;
int max_nr_timesteps = 1024;
double cellSize = 1.0 / 3600.0 * Math.PI / 180.0;
var griddingConstants = new GriddingConstants(data.VisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
var obsName = "simulated";
var lambda = 0.5f;
var alpha = 0.2f;
var epsilon = 1e-5f;
var maxMajorCycles = 5;
MajorCycle.ReconstructSerialCD(obsName, data, griddingConstants, true, 1, maxMajorCycles, lambda, alpha, 5000, epsilon);
MajorCycle.ReconstructPCDM(obsName, data, griddingConstants, 1, maxMajorCycles, 1, lambda, alpha, 30, epsilon);
}
public static void RunSimulated(string[] args)
{
using (var env = new MPI.Environment(ref args, MPI.Threading.Serialized))
{
var proc = Process.GetCurrentProcess();
var name = proc.ProcessName;
Console.WriteLine(" name: " + name);
//System.Threading.Thread.Sleep(17000);
var comm = Communicator.world;
//READ DATA
var folder = @"C:\dev\GitHub\p9-data\small\fits\simulation_point\";
var bla = @"C:\dev\GitHub\p9-data\small\fits\simulation_point";
var fullData = DistributedData.LoadSimulated(folder);
var data = DistributedData.SplitDataAmongNodes(comm, fullData);
var totalVisibilitiesCount = fullData.VisibilitiesCount;
int gridSize = 256;
int subgridsize = 16;
int kernelSize = 8;
int max_nr_timesteps = 512;
double cellSize = 1.0 / 3600.0 * PI / 180.0;
var c = new GriddingConstants(totalVisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
comm.Barrier();
if (comm.Rank == 0)
{
Console.WriteLine("Done Reading, Starting reconstruction");
}
var reconstruction = MPIMajorCycle.Reconstruct(comm, data, c, 1, 0.5f, 0.8f, 1000);
if (comm.Rank == 0)
{
FitsIO.Write(reconstruction, "simulatedReconstruction.fits");
}
}
}
public static void RunTinyMeerKAT(string[] args)
{
using (var env = new MPI.Environment(ref args, MPI.Threading.Serialized))
{
var proc = Process.GetCurrentProcess();
var name = proc.ProcessName;
Console.WriteLine(" name: " + name);
var comm = Communicator.world;
var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\";
var data = DistributedData.LoadTinyMeerKAT2(comm.Rank, comm.Size, folder);
var totalVisCount = comm.Allreduce(data.VisibilitiesCount, (x, y) => x + y);
int gridSize = 3072;
int subgridsize = 32;
int kernelSize = 16;
int max_nr_timesteps = 1024;
double cellSize = 1.5 / 3600.0 * PI / 180.0;
var c = new GriddingConstants(totalVisCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
comm.Barrier();
if (comm.Rank == 0)
{
Console.WriteLine("Done Reading, Starting reconstruction");
}
var lambda = 0.4f;
var alpha = 0.1f;
var reconstruction = MPIMajorCycle.Reconstruct(comm, data, c, 2, lambda, alpha, 10000);
if (comm.Rank == 0)
{
FitsIO.Write(reconstruction, "tinyMeerKATReconstruction.fits");
}
}
}
public static float[,] Reconstruct(Intracommunicator comm, DistributedData.LocalDataset local, GriddingConstants c, int maxCycle, float lambda, float alpha, int iterPerCycle = 1000, bool usePathDeconvolution = false)
{
var watchTotal = new Stopwatch();
var watchForward = new Stopwatch();
var watchBackward = new Stopwatch();
var watchDeconv = new Stopwatch();
watchTotal.Start();
var metadata = Partitioner.CreatePartition(c, local.UVW, local.Frequencies);
var patchSize = CalculateLocalImageSection(comm.Rank, comm.Size, c.GridSize, c.GridSize);
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
//calculate psf and prepare for correlation in the Fourier space
var psf = CalculatePSF(comm, c, metadata, local.UVW, local.Flags, local.Frequencies);
Complex[,] PsfCorrelation = null;
var maxSidelobe = PSF.CalcMaxSidelobe(psf);
lambda = (float)(lambda * PSF.CalcMaxLipschitz(psf));
StreamWriter writer = null;
if (comm.Rank == 0)
{
FitsIO.Write(psf, "psf.fits");
Console.WriteLine("done PSF gridding ");
PsfCorrelation = PSF.CalcPaddedFourierCorrelation(psf, totalSize);
writer = new StreamWriter(comm.Size + "runtimestats.txt");
}
var deconvovler = new MPIGreedyCD(comm, totalSize, patchSize, psf);
var residualVis = local.Visibilities;
var xLocal = new float[patchSize.YEnd - patchSize.Y, patchSize.XEnd - patchSize.X];
for (int cycle = 0; cycle < maxCycle; cycle++)
{
if (comm.Rank == 0)
{
Console.WriteLine("cycle " + cycle);
}
var dirtyImage = ForwardCalculateB(comm, c, metadata, residualVis, local.UVW, local.Frequencies, PsfCorrelation, psf, maxSidelobe, watchForward);
var bLocal = GetImgSection(dirtyImage.Image, patchSize);
MPIGreedyCD.Statistics lastRun;
if (usePathDeconvolution)
{
var currentLambda = Math.Max(1.0f / alpha * dirtyImage.MaxSidelobeLevel, lambda);
lastRun = deconvovler.DeconvolvePath(xLocal, bLocal, currentLambda, 4.0f, alpha, 5, iterPerCycle, 2e-5f);
}
else
{
lastRun = deconvovler.Deconvolve(xLocal, bLocal, lambda, alpha, iterPerCycle, 1e-5f);
}
if (comm.Rank == 0)
{
WriteToFile(cycle, lastRun, writer);
if (lastRun.Converged)
{
Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
}
else
{
Console.WriteLine("-------------------------------not converged----------------------");
}
}
comm.Barrier();
if (comm.Rank == 0)
{
watchDeconv.Stop();
}
float[][,] totalX = null;
comm.Gather(xLocal, 0, ref totalX);
Complex[,] modelGrid = null;
if (comm.Rank == 0)
{
watchBackward.Start();
var x = new float[c.GridSize, c.GridSize];
StitchImage(totalX, x, comm.Size);
FitsIO.Write(x, "xImage_" + cycle + ".fits");
FFT.Shift(x);
modelGrid = FFT.Forward(x);
}
comm.Broadcast(ref modelGrid, 0);
var modelVis = IDG.DeGrid(c, metadata, modelGrid, local.UVW, local.Frequencies);
residualVis = Visibilities.Substract(local.Visibilities, modelVis, local.Flags);
}
writer.Close();
float[][,] gatherX = null;
comm.Gather(xLocal, 0, ref gatherX);
float[,] reconstructed = null;
if (comm.Rank == 0)
{
reconstructed = new float[c.GridSize, c.GridSize];;
StitchImage(gatherX, reconstructed, comm.Size);
}
return(reconstructed);
}
public static void RunProcessorComparison()
{
var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\";
var data = MeasurementData.LoadLMC(folder);
int gridSize = 3072;
int subgridsize = 32;
int kernelSize = 16;
int max_nr_timesteps = 1024;
double cellSize = 1.5 / 3600.0 * Math.PI / 180.0;
int wLayerCount = 24;
var maxW = 0.0;
for (int i = 0; i < data.UVW.GetLength(0); i++)
{
for (int j = 0; j < data.UVW.GetLength(1); j++)
{
maxW = Math.Max(maxW, Math.Abs(data.UVW[i, j, 2]));
}
}
maxW = Partitioner.MetersToLambda(maxW, data.Frequencies[data.Frequencies.Length - 1]);
var visCount2 = 0;
for (int i = 0; i < data.Flags.GetLength(0); i++)
{
for (int j = 0; j < data.Flags.GetLength(1); j++)
{
for (int k = 0; k < data.Flags.GetLength(2); k++)
{
if (!data.Flags[i, j, k])
{
visCount2++;
}
}
}
}
double wStep = maxW / (wLayerCount);
var c = new GriddingConstants(data.VisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep);
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];
for (int i = 0; i < data.Visibilities.GetLength(0); i++)
{
for (int j = 0; j < data.Visibilities.GetLength(1); j++)
{
for (int k = 0; k < data.Visibilities.GetLength(2); k++)
{
if (!data.Flags[i, j, k])
{
psfVis[i, j, k] = new Complex(1.0, 0);
}
else
{
psfVis[i, j, k] = new Complex(0, 0);
}
}
}
}
Console.WriteLine("gridding psf");
var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies);
var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
Directory.CreateDirectory("PCDMComparison/processors");
var processorCount = new int[] { 1, 4, 8, 16, 32 };
foreach (var count in processorCount)
{
ReconstructPCDM(data, c, psf, "PCDMComparison/processors", "pcdm" + count, 3, 0.1f, count);
ReconstructSerial(data, c, psf, "PCDMComparison/processors", "serial" + count, count);
}
}
public static void DebugSimulatedMixed()
{
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_mixed\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_mixed\uvw.fits");
var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_mixed\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
var visibilitiesCount = visibilities.Length;
int gridSize = 1024;
int subgridsize = 16;
int kernelSize = 4;
//cell = image / grid
int max_nr_timesteps = 512;
double scaleArcSec = 0.5 / 3600.0 * PI / 180.0;
var watchTotal = new Stopwatch();
var watchForward = new Stopwatch();
var watchBackwards = new Stopwatch();
var watchDeconv = new Stopwatch();
watchTotal.Start();
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)scaleArcSec, 1, 0.0f);
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psf = IDG.CalculatePSF(c, metadata, uvw, flags, frequencies);
FitsIO.Write(psf, "psf.fits");
var psfCut = CutImg(psf, 2);
FitsIO.Write(psfCut, "psfCut.fits");
var maxSidelobe = CommonDeprecated.PSF.CalcMaxSidelobe(psf);
var xImage = new double[gridSize, gridSize];
var residualVis = visibilities;
var maxCycle = 10;
for (int cycle = 0; cycle < maxCycle; cycle++)
{
watchForward.Start();
var dirtyImage = IDG.ToImage(c, metadata, residualVis, uvw, frequencies);
watchForward.Stop();
FitsIO.Write(dirtyImage, "dirty" + cycle + ".fits");
watchDeconv.Start();
var sideLobe = maxSidelobe * GetMax(dirtyImage);
Console.WriteLine("sideLobeLevel: " + sideLobe);
var PsfCorrelation = CommonDeprecated.PSF.CalculateFourierCorrelation(psfCut, c.GridSize, c.GridSize);
var b = CommonDeprecated.Residuals.CalculateBMap(dirtyImage, PsfCorrelation, psfCut.GetLength(0), psfCut.GetLength(1));
var lambda = 100.0;
var alpha = 0.95;
var currentLambda = Math.Max(1.0 / alpha * sideLobe, lambda);
var converged = SerialCDReference.DeconvolvePath(xImage, b, psfCut, currentLambda, 5.0, alpha, 5, 6000, 1e-3);
//var converged = GreedyCD2.Deconvolve(xImage, b, psfCut, currentLambda, alpha, 5000);
if (converged)
{
Console.WriteLine("-----------------------------CONVERGED!!!! with lambda " + currentLambda + "------------------------");
}
else
{
Console.WriteLine("-------------------------------not converged with lambda " + currentLambda + "----------------------");
}
watchDeconv.Stop();
FitsIO.Write(xImage, "xImage_" + cycle + ".fits");
watchBackwards.Start();
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
residualVis = Visibilities.Substract(visibilities, modelVis, flags);
watchBackwards.Stop();
}
watchBackwards.Stop();
watchTotal.Stop();
var timetable = "total elapsed: " + watchTotal.Elapsed;
timetable += "\n" + "idg forward elapsed: " + watchForward.Elapsed;
timetable += "\n" + "idg backwards elapsed: " + watchBackwards.Elapsed;
timetable += "\n" + "devonvolution: " + watchDeconv.Elapsed;
File.WriteAllText("watches_single.txt", timetable);
}
public static void MeerKATFull()
{
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw0.fits");
var flags = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length);
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
for (int i = 1; i < 8; i++)
{
var uvw0 = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw" + i + ".fits");
var flags0 = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length);
var visibilities0 = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, norm);
uvw = FitsIO.Stitch(uvw, uvw0);
flags = FitsIO.Stitch(flags, flags0);
visibilities = FitsIO.Stitch(visibilities, visibilities0);
}
/*
* var frequencies = FitsIO.ReadFrequencies(@"freq.fits");
* var uvw = FitsIO.ReadUVW("uvw0.fits");
* var flags = FitsIO.ReadFlags("flags0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length);
* double norm = 2.0;
* var visibilities = FitsIO.ReadVisibilities("vis0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
*/
var visCount2 = 0;
for (int i = 0; i < flags.GetLength(0); i++)
{
for (int j = 0; j < flags.GetLength(1); j++)
{
for (int k = 0; k < flags.GetLength(2); k++)
{
if (!flags[i, j, k])
{
visCount2++;
}
}
}
}
var visibilitiesCount = visCount2;
int gridSize = 1024;
int subgridsize = 16;
int kernelSize = 4;
//cell = image / grid
int max_nr_timesteps = 512;
double scaleArcSec = 2.5 / 3600.0 * PI / 180.0;
var watchTotal = new Stopwatch();
var watchForward = new Stopwatch();
var watchBackwards = new Stopwatch();
var watchDeconv = new Stopwatch();
watchTotal.Start();
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)scaleArcSec, 1, 0.0f);
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psf = IDG.CalculatePSF(c, metadata, uvw, flags, frequencies);
FitsIO.Write(psf, "psf.fits");
var psfCut = CutImg(psf, 2);
FitsIO.Write(psfCut, "psfCut.fits");
var maxSidelobe = CommonDeprecated.PSF.CalcMaxSidelobe(psf);
var psfCorrelated = CommonDeprecated.PSF.CalculateFourierCorrelation(psfCut, c.GridSize, c.GridSize);
var xImage = new double[gridSize, gridSize];
var residualVis = visibilities;
var maxCycle = 2;
for (int cycle = 0; cycle < maxCycle; cycle++)
{
watchForward.Start();
var dirtyImage = IDG.ToImage(c, metadata, residualVis, uvw, frequencies);
watchForward.Stop();
FitsIO.Write(dirtyImage, "dirty" + cycle + ".fits");
watchDeconv.Start();
var sideLobe = maxSidelobe * GetMax(dirtyImage);
Console.WriteLine("sideLobeLevel: " + sideLobe);
var b = CommonDeprecated.Residuals.CalculateBMap(dirtyImage, psfCorrelated, psfCut.GetLength(0), psfCut.GetLength(1));
var lambda = 0.8;
var alpha = 0.05;
var currentLambda = Math.Max(1.0 / alpha * sideLobe, lambda);
var converged = SerialCDReference.DeconvolvePath(xImage, b, psfCut, currentLambda, 4.0, alpha, 5, 1000, 2e-5);
//var converged = GreedyCD2.Deconvolve(xImage, b, psfCut, currentLambda, alpha, 5000);
if (converged)
{
Console.WriteLine("-----------------------------CONVERGED!!!! with lambda " + currentLambda + "------------------------");
}
else
{
Console.WriteLine("-------------------------------not converged with lambda " + currentLambda + "----------------------");
}
watchDeconv.Stop();
FitsIO.Write(xImage, "xImage_" + cycle + ".fits");
watchBackwards.Start();
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
residualVis = Visibilities.Substract(visibilities, modelVis, flags);
watchBackwards.Stop();
}
watchBackwards.Stop();
watchTotal.Stop();
var timetable = "total elapsed: " + watchTotal.Elapsed;
timetable += "\n" + "idg forward elapsed: " + watchForward.Elapsed;
timetable += "\n" + "idg backwards elapsed: " + watchBackwards.Elapsed;
timetable += "\n" + "devonvolution: " + watchDeconv.Elapsed;
File.WriteAllText("watches_single.txt", timetable);
}
public static void DebugSimulatedApprox()
{
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits");
var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
var visibilitiesCount = visibilities.Length;
int gridSize = 256;
int subgridsize = 8;
int kernelSize = 4;
int max_nr_timesteps = 1024;
double cellSize = 1.0 / 3600.0 * PI / 180.0;
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
var watchTotal = new Stopwatch();
var watchForward = new Stopwatch();
var watchBackwards = new Stopwatch();
var watchDeconv = new Stopwatch();
watchTotal.Start();
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var psfCut = PSF.Cut(psf);
FitsIO.Write(psfCut, "psfCut.fits");
var random = new Random(123);
var totalSize = new Rectangle(0, 0, gridSize, gridSize);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
var fastCD = new FastSerialCD(totalSize, psfCut);
//fastCD.ResetAMap(psf);
var lambda = 0.5f * fastCD.MaxLipschitz;
var alpha = 0.8f;
var approx = new ApproxParallel();
var approx2 = new ApproxFast(totalSize, psfCut, 4, 8, 0f, 0.25f, false, true);
var xImage = new float[gridSize, gridSize];
var residualVis = visibilities;
/*var truth = new double[gridSize, gridSize];
* truth[30, 30] = 1.0;
* truth[35, 36] = 1.5;
* var truthVis = IDG.ToVisibilities(c, metadata, truth, uvw, frequencies);
* visibilities = truthVis;
* var residualVis = truthVis;*/
var data = new ApproxFast.TestingData(new StreamWriter("approxConvergence.txt"));
for (int cycle = 0; cycle < 4; cycle++)
{
//FORWARD
watchForward.Start();
var dirtyGrid = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, "dirty_" + cycle + ".fits");
watchForward.Stop();
//DECONVOLVE
watchDeconv.Start();
//approx.ISTAStep(xImage, dirtyImage, psf, lambda, alpha);
//FitsIO.Write(xImage, "xIsta.fits");
//FitsIO.Write(dirtyImage, "dirtyFista.fits");
//bMapCalculator.ConvolveInPlace(dirtyImage);
//FitsIO.Write(dirtyImage, "bMap_" + cycle + ".fits");
//var result = fastCD.Deconvolve(xImage, dirtyImage, 0.5f * fastCD.MaxLipschitz, 0.8f, 1000, 1e-4f);
//var converged = approx.DeconvolveActiveSet(xImage, dirtyImage, psfCut, lambda, alpha, random, 8, 1, 1);
//var converged = approx.DeconvolveGreedy(xImage, dirtyImage, psfCut, lambda, alpha, random, 4, 4, 500);
//var converged = approx.DeconvolveApprox(xImage, dirtyImage, psfCut, lambda, alpha, random, 1, threads, 500, 1e-4f, cycle == 0);
approx2.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, psf, lambda, alpha, random, 10, 1e-4f);
if (data.converged)
{
Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
}
else
{
Console.WriteLine("-------------------------------not converged----------------------");
}
FitsIO.Write(xImage, "xImageApprox_" + cycle + ".fits");
watchDeconv.Stop();
//BACKWARDS
watchBackwards.Start();
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
residualVis = Visibilities.Substract(visibilities, modelVis, flags);
watchBackwards.Stop();
}
var dirtyGridCheck = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
var dirtyCheck = FFT.Backward(dirtyGridCheck, c.VisibilitiesCount);
FFT.Shift(dirtyCheck);
var l2Penalty = Residuals.CalcPenalty(ToFloatImage(dirtyCheck));
var elasticPenalty = ElasticNet.CalcPenalty(xImage, (float)lambda, (float)alpha);
var sum = l2Penalty + elasticPenalty;
data.writer.Close();
}
public static void Run()
{
var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\";
var data = MeasurementData.LoadLMC(folder);
int gridSize = 3072;
int subgridsize = 32;
int kernelSize = 16;
int max_nr_timesteps = 1024;
double cellSize = 1.5 / 3600.0 * PI / 180.0;
int wLayerCount = 24;
var maxW = 0.0;
for (int i = 0; i < data.UVW.GetLength(0); i++)
{
for (int j = 0; j < data.UVW.GetLength(1); j++)
{
maxW = Math.Max(maxW, Math.Abs(data.UVW[i, j, 2]));
}
}
maxW = Partitioner.MetersToLambda(maxW, data.Frequencies[data.Frequencies.Length - 1]);
double wStep = maxW / (wLayerCount);
var c = new GriddingConstants(data.VisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep);
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];
for (int i = 0; i < data.Visibilities.GetLength(0); i++)
{
for (int j = 0; j < data.Visibilities.GetLength(1); j++)
{
for (int k = 0; k < data.Visibilities.GetLength(2); k++)
{
if (!data.Flags[i, j, k])
{
psfVis[i, j, k] = new Complex(1.0, 0);
}
else
{
psfVis[i, j, k] = new Complex(0, 0);
}
}
}
}
Console.WriteLine("gridding psf");
var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies);
var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var outFolder = "ApproxExperiment";
Directory.CreateDirectory(outFolder);
FitsIO.Write(psf, Path.Combine(outFolder, "psfFull.fits"));
//tryout with simply cutting the PSF
RunPsfSize(data, c, psf, outFolder);
//RunBlocksize(data, c, psf, outFolder);
RunSearchPercent(data, c, psf, outFolder);
//RunNotAccelerated(data, c, psf, outFolder);
}
private static void ReconstructMinorCycle(MeasurementData input, GriddingConstants c, int cutFactor, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, bool useAccelerated = true, int blockSize = 1, int maxCycle = 6)
{
var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(fullPsf, cutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor);
var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2);
var random = new Random(123);
var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.1f, searchPercent, false, useAccelerated);
using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf))
{
var currentBMapCalculator = bMapCalculator;
var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * maxLipschitz);
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
var alpha = ALPHA;
ApproxFast.LAMBDA_TEST = lambdaTrue;
ApproxFast.ALPHA_TEST = alpha;
var switchedToOtherPsf = false;
var writer = new StreamWriter(folder + "/" + file + "_lambda.txt");
var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt"));
var xImage = new float[c.GridSize, c.GridSize];
var residualVis = input.Visibilities;
for (int cycle = 0; cycle < maxCycle; cycle++)
{
Console.WriteLine("cycle " + cycle);
var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");
var minLambda = 0.0f;
var dirtyCopy = Copy(dirtyImage);
var xCopy = Copy(xImage);
var currentLambda = 0f;
//var residualsConvolver = new PaddedConvolver(PSF.CalcPaddedFourierConvolution(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)));
for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++)
{
FitsIO.Write(dirtyImage, folder + "/dirtyMinor_" + minorCycle + ".fits");
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = currentBMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);
if (minorCycle == 0)
{
minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
}
if (currentLambda < minLambda)
{
currentLambda = minLambda;
}
writer.WriteLine(cycle + ";" + minorCycle + ";" + currentLambda + ";" + minLambda);
writer.Flush();
approx.DeconvolveTest(data, cycle, minorCycle, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f);
FitsIO.Write(xImage, folder + "/xImageMinor_" + minorCycle + ".fits");
if (currentLambda == lambda | currentLambda == minLambda)
{
break;
}
Console.WriteLine("resetting residuals!!");
//reset dirtyImage with full PSF
var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
}
});
residualsConvolver.ConvolveInPlace(residualsUpdate);
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
}
});
}
if (currentLambda == lambda & !switchedToOtherPsf)
{
approx.ResetAMap(fullPsf);
currentBMapCalculator = bMapCalculator2;
lambda = lambdaTrue;
switchedToOtherPsf = true;
writer.WriteLine("switched");
writer.Flush();
}
FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits");
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
}
writer.Close();
}
}
public static void DebugILGPU()
{
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits");
var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\vis.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
var visibilitiesCount = visibilities.Length;
int gridSize = 256;
int subgridsize = 8;
int kernelSize = 4;
int max_nr_timesteps = 1024;
double cellSize = 1.0 / 3600.0 * PI / 180.0;
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
var watchTotal = new Stopwatch();
var watchForward = new Stopwatch();
var watchBackwards = new Stopwatch();
var watchDeconv = new Stopwatch();
watchTotal.Start();
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
var psf = FFT.Backward(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var psfCutDouble = CutImg(psf);
var psfCut = ToFloatImage(psfCutDouble);
FitsIO.Write(psfCut, "psfCut.fits");
var totalSize = new Rectangle(0, 0, gridSize, gridSize);
var imageSection = new Rectangle(0, 128, gridSize, gridSize);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)));
var fastCD = new FastSerialCD(totalSize, psfCut);
fastCD.ResetLipschitzMap(ToFloatImage(psf));
var gpuCD = new GPUSerialCD(totalSize, psfCut, 100);
var lambda = 0.5f * fastCD.MaxLipschitz;
var alpha = 0.8f;
var xImage = new float[gridSize, gridSize];
var residualVis = visibilities;
/*var truth = new double[gridSize, gridSize];
* truth[30, 30] = 1.0;
* truth[35, 36] = 1.5;
* var truthVis = IDG.ToVisibilities(c, metadata, truth, uvw, frequencies);
* visibilities = truthVis;
* var residualVis = truthVis;*/
for (int cycle = 0; cycle < 4; cycle++)
{
//FORWARD
watchForward.Start();
var dirtyGrid = IDG.Grid(c, metadata, residualVis, uvw, frequencies);
var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, "dirty_" + cycle + ".fits");
watchForward.Stop();
//DECONVOLVE
watchDeconv.Start();
bMapCalculator.ConvolveInPlace(dirtyImage);
FitsIO.Write(dirtyImage, "bMap_" + cycle + ".fits");
//var result = fastCD.Deconvolve(xImage, dirtyImage, lambda, alpha, 1000, 1e-4f);
var result = gpuCD.Deconvolve(xImage, dirtyImage, lambda, alpha, 1000, 1e-4f);
if (result.Converged)
{
Console.WriteLine("-----------------------------CONVERGED!!!!------------------------");
}
else
{
Console.WriteLine("-------------------------------not converged----------------------");
}
FitsIO.Write(xImage, "xImageGreedy" + cycle + ".fits");
FitsIO.Write(dirtyImage, "residualDebug_" + cycle + ".fits");
watchDeconv.Stop();
//BACKWARDS
watchBackwards.Start();
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGrid(c, metadata, xGrid, uvw, frequencies);
residualVis = Visibilities.Substract(visibilities, modelVis, flags);
watchBackwards.Stop();
var hello = FFT.Forward(xImage, 1.0);
hello = Common.Fourier2D.Multiply(hello, psfGrid);
var hImg = FFT.Backward(hello, (double)(128 * 128));
//FFT.Shift(hImg);
FitsIO.Write(hImg, "modelDirty_FFT.fits");
var imgRec = IDG.ToImage(c, metadata, modelVis, uvw, frequencies);
FitsIO.Write(imgRec, "modelDirty" + cycle + ".fits");
}
}
public static void DebugdWStack()
{
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw0.fits");
var flags = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length);
double norm = 2.0;
var visibilities = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis0.fits", uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm);
for (int i = 1; i < 8; i++)
{
var uvw0 = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\uvw" + i + ".fits");
var flags0 = FitsIO.ReadFlags(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\flags" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length);
var visibilities0 = FitsIO.ReadVisibilities(@"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\vis" + i + ".fits", uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, norm);
uvw = FitsIO.Stitch(uvw, uvw0);
flags = FitsIO.Stitch(flags, flags0);
visibilities = FitsIO.Stitch(visibilities, visibilities0);
}
var maxW = 0.0;
for (int i = 0; i < uvw.GetLength(0); i++)
{
for (int j = 0; j < uvw.GetLength(1); j++)
{
maxW = Math.Max(maxW, Math.Abs(uvw[i, j, 2]));
}
}
maxW = Partitioner.MetersToLambda(maxW, frequencies[frequencies.Length - 1]);
var visCount2 = 0;
for (int i = 0; i < flags.GetLength(0); i++)
{
for (int j = 0; j < flags.GetLength(1); j++)
{
for (int k = 0; k < flags.GetLength(2); k++)
{
if (!flags[i, j, k])
{
visCount2++;
}
}
}
}
var visibilitiesCount = visCount2;
int gridSize = 4096;
int subgridsize = 16;
int kernelSize = 8;
int max_nr_timesteps = 1024;
double cellSize = 1.6 / 3600.0 * PI / 180.0;
int wLayerCount = 32;
double wStep = maxW / (wLayerCount);
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep);
var c2 = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0);
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psfVis = new Complex[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length];
for (int i = 0; i < visibilities.GetLength(0); i++)
{
for (int j = 0; j < visibilities.GetLength(1); j++)
{
for (int k = 0; k < visibilities.GetLength(2); k++)
{
if (!flags[i, j, k])
{
psfVis[i, j, k] = new Complex(1.0, 0);
}
else
{
psfVis[i, j, k] = new Complex(0, 0);
}
}
}
}
var psfGrid = IDG.GridW(c, metadata, psfVis, uvw, frequencies);
var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
FitsIO.Write(psf, "psfWStack.fits");
var totalSize = new Rectangle(0, 0, gridSize, gridSize);
var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1)));
var fastCD = new FastSerialCD(totalSize, psf);
var lambda = 0.4f * fastCD.MaxLipschitz;
var alpha = 0.1f;
var xImage = new float[gridSize, gridSize];
var residualVis = visibilities;
for (int cycle = 0; cycle < 8; cycle++)
{
var dirtyGrid = IDG.GridW(c, metadata, residualVis, uvw, frequencies);
var dirty = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirty);
FitsIO.Write(dirty, "dirty_" + cycle + ".fits");
bMapCalculator.ConvolveInPlace(dirty);
FitsIO.Write(dirty, "bMap_" + cycle + ".fits");
var result = fastCD.Deconvolve(xImage, dirty, lambda, alpha, 10000, 1e-4f);
FitsIO.Write(xImage, "xImageGreedy" + cycle + ".fits");
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, uvw, frequencies);
var modelGrid = IDG.GridW(c, metadata, modelVis, uvw, frequencies);
var model = FFT.WStackIFFTFloat(modelGrid, c.VisibilitiesCount);
FFT.Shift(model);
FitsIO.Write(model, "model_" + cycle + ".fits");
residualVis = Visibilities.Substract(visibilities, modelVis, flags);
}
}
private static void ReconstructPCDM(MeasurementData input, GriddingConstants c, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, int processorCount)
{
var totalWatch = new Stopwatch();
var currentWatch = new Stopwatch();
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(fullPsf, CUT_FACTOR_PCDM);
var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, CUT_FACTOR_PCDM);
var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2);
var random = new Random(123);
var pcdm = new ParallelCoordinateDescent(totalSize, psfCut, 1, 1000, searchPercent);
var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);
using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf))
{
var currentBMapCalculator = bMapCalculator;
var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * maxLipschitz);
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
var alpha = ALPHA;
var switchedToOtherPsf = false;
var writer = new StreamWriter(folder + "/" + file + ".txt");
var xImage = new float[c.GridSize, c.GridSize];
var residualVis = input.Visibilities;
ParallelCoordinateDescent.PCDMStatistics lastResult = null;
for (int cycle = 0; cycle < 6; cycle++)
{
Console.WriteLine("Beginning Major cycle " + cycle);
var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");
currentWatch.Restart();
totalWatch.Start();
var breakMajor = false;
var minLambda = 0.0f;
var dirtyCopy = Copy(dirtyImage);
var xCopy = Copy(xImage);
var currentLambda = 0f;
var currentObjective = 0.0;
var absMax = 0.0f;
for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++)
{
Console.WriteLine("Beginning Minor Cycle " + minorCycle);
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = currentBMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda);
if (minorCycle == 0)
{
minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
}
if (currentLambda < minLambda)
{
currentLambda = minLambda;
}
currentObjective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
absMax = pcdm.GetAbsMax(xImage, bMap, lambdaTrue, alpha);
if (absMax < MAJOR_STOP)
{
breakMajor = true;
break;
}
lastResult = pcdm.Deconvolve(xImage, bMap, currentLambda, alpha, 100, 1e-5f);
if (currentLambda == lambda | currentLambda == minLambda)
{
break;
}
var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
}
});
residualsConvolver.ConvolveInPlace(residualsUpdate);
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
}
});
}
currentWatch.Stop();
totalWatch.Stop();
writer.WriteLine(cycle + ";" + currentLambda + ";" + currentObjective + ";" + absMax + ";" + lastResult.IterationCount + ";" + totalWatch.Elapsed.TotalSeconds + ";" + currentWatch.Elapsed.TotalSeconds);
writer.Flush();
FitsIO.Write(xImage, folder + "/xImage_pcdm_" + cycle + ".fits");
if (breakMajor)
{
break;
}
if (currentLambda == lambda & !switchedToOtherPsf)
{
pcdm.ResetAMap(fullPsf);
currentBMapCalculator = bMapCalculator2;
lambda = lambdaTrue;
switchedToOtherPsf = true;
writer.WriteLine("switched");
writer.Flush();
}
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
}
writer.Close();
}
}
/// <summary>
/// Major cycle implemnentation for the parallel coordinate descent algorithm
/// </summary>
/// <param name="data"></param>
/// <param name="c"></param>
/// <param name="psfCutFactor"></param>
/// <param name="maxMajorCycle"></param>
/// <param name="maxMinorCycle"></param>
/// <param name="lambda"></param>
/// <param name="alpha"></param>
/// <param name="deconvIterations"></param>
/// <param name="deconvEpsilon"></param>
public static void ReconstructPCDM(string obsName, MeasurementData data, GriddingConstants c, int psfCutFactor, int maxMajorCycle, int maxMinorCycle, float lambda, float alpha, int deconvIterations, float deconvEpsilon)
{
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];
for (int i = 0; i < data.Visibilities.GetLength(0); i++)
{
for (int j = 0; j < data.Visibilities.GetLength(1); j++)
{
for (int k = 0; k < data.Visibilities.GetLength(2); k++)
{
if (!data.Flags[i, j, k])
{
psfVis[i, j, k] = new Complex(1.0, 0);
}
else
{
psfVis[i, j, k] = new Complex(0, 0);
}
}
}
}
Console.WriteLine("gridding psf");
var psfGrid = IDG.Grid(c, metadata, psfVis, data.UVW, data.Frequencies);
var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var totalWatch = new Stopwatch();
var currentWatch = new Stopwatch();
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(psf, psfCutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(psf, psfCutFactor);
var sidelobeHalf = PSF.CalcMaxSidelobe(psf, 2);
var pcdm = new ParallelCoordinateDescent(totalSize, psfCut, Environment.ProcessorCount, 1000);
using (var gCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))))
using (var gCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))))
using (var residualsConvolver = new PaddedConvolver(totalSize, psf))
{
var currentGCalculator = gCalculator;
var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
var lambdaLipschitz = (float)(lambda * maxLipschitz);
var lambdaTrue = (float)(lambda * PSF.CalcMaxLipschitz(psf));
var switchedToOtherPsf = false;
var xImage = new float[c.GridSize, c.GridSize];
var residualVis = data.Visibilities;
ParallelCoordinateDescent.PCDMStatistics lastResult = null;
for (int cycle = 0; cycle < maxMajorCycle; cycle++)
{
Console.WriteLine("Beginning Major cycle " + cycle);
var dirtyGrid = IDG.GridW(c, metadata, residualVis, data.UVW, data.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, obsName + "_dirty_pcdm_majorCycle" + cycle + ".fits");
currentWatch.Restart();
totalWatch.Start();
var breakMajor = false;
var minLambda = 0.0f;
var dirtyCopy = Copy(dirtyImage);
var xCopy = Copy(xImage);
var currentLambda = 0f;
var currentObjective = 0.0;
var absMax = 0.0f;
for (int minorCycle = 0; minorCycle < maxMinorCycle; minorCycle++)
{
Console.WriteLine("Beginning Minor Cycle " + minorCycle);
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = currentGCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
currentLambda = (float)Math.Max(currentSideLobe / alpha, lambdaLipschitz);
if (minorCycle == 0)
{
minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha);
}
if (currentLambda < minLambda)
{
currentLambda = minLambda;
}
currentObjective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
absMax = pcdm.GetAbsMax(xImage, bMap, lambdaTrue, alpha);
if (absMax < MAJOR_EPSILON)
{
breakMajor = true;
break;
}
lastResult = pcdm.Deconvolve(xImage, bMap, currentLambda, alpha, 40, deconvEpsilon);
if (currentLambda == lambda | currentLambda == minLambda)
{
break;
}
var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)];
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j];
}
});
residualsConvolver.ConvolveInPlace(residualsUpdate);
Parallel.For(0, xCopy.GetLength(0), (i) =>
{
for (int j = 0; j < xCopy.GetLength(1); j++)
{
dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j];
}
});
}
currentWatch.Stop();
totalWatch.Stop();
if (breakMajor)
{
break;
}
if (currentLambda == lambda & !switchedToOtherPsf)
{
pcdm.ResetAMap(psf);
currentGCalculator = gCalculator2;
lambda = lambdaTrue;
switchedToOtherPsf = true;
}
FitsIO.Write(xImage, obsName + "_model_pcdm_majorCycle" + cycle + ".fits");
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, data.UVW, data.Frequencies);
residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
}
Console.WriteLine("Reconstruction finished in (seconds): " + totalWatch.Elapsed.TotalSeconds);
}
}
public static void GenerateCLEANExample(string simulatedLocation, string outputFolder)
{
var data = MeasurementData.LoadSimulatedPoints(simulatedLocation);
var cellSize = 1.0 / 3600.0 * Math.PI / 180.0;
var c = new GriddingConstants(data.VisibilitiesCount, 256, 8, 4, 512, (float)cellSize, 1, 0.0);
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfGrid = IDG.GridPSF(c, metadata, data.UVW, data.Flags, data.Frequencies);
var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
Directory.CreateDirectory(outputFolder);
var reconstruction = new float[c.GridSize, c.GridSize];
var residualVis = data.Visibilities;
for (int cycle = 0; cycle < 10; cycle++)
{
Console.WriteLine("in cycle " + cycle);
var dirtyGrid = IDG.Grid(c, metadata, residualVis, data.UVW, data.Frequencies);
var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
//FitsIO.Write(dirtyImage, Path.Combine(outputFolder, "dirty_CLEAN_" + cycle + ".fits"));
Tools.WriteToMeltCSV(dirtyImage, Path.Combine(outputFolder, "dirty_CLEAN_" + cycle + ".csv"));
var maxY = -1;
var maxX = -1;
var max = 0.0f;
for (int y = 0; y < dirtyImage.GetLength(0); y++)
{
for (int x = 0; x < dirtyImage.GetLength(1); x++)
{
if (max < Math.Abs(dirtyImage[y, x]))
{
maxY = y;
maxX = x;
max = Math.Abs(dirtyImage[y, x]);
}
}
}
//FitsIO.Write(reconstruction, Path.Combine(outputFolder, "model_CLEAN_" + cycle + ".fits"));
Tools.WriteToMeltCSV(PSF.Cut(reconstruction), Path.Combine(outputFolder, "model_CLEAN_" + cycle + ".csv"));
reconstruction[maxY, maxX] += 0.5f * dirtyImage[maxY, maxX];
FFT.Shift(reconstruction);
var xGrid = FFT.Forward(reconstruction);
FFT.Shift(reconstruction);
var modelVis = IDG.DeGrid(c, metadata, xGrid, data.UVW, data.Frequencies);
residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
}
var cleanbeam = new float[c.GridSize, c.GridSize];
var x0 = c.GridSize / 2;
var y0 = c.GridSize / 2;
for (int y = 0; y < cleanbeam.GetLength(0); y++)
{
for (int x = 0; x < cleanbeam.GetLength(1); x++)
{
cleanbeam[y, x] = (float)(1.0 * Math.Exp(-(Math.Pow(x0 - x, 2) / 16 + Math.Pow(y0 - y, 2) / 16)));
}
}
FitsIO.Write(cleanbeam, Path.Combine(outputFolder, "clbeam.fits"));
FFT.Shift(cleanbeam);
var CL = FFT.Forward(cleanbeam);
var REC = FFT.Forward(reconstruction);
var CONF = Common.Fourier2D.Multiply(REC, CL);
var cleaned = FFT.BackwardFloat(CONF, reconstruction.Length);
//FFT.Shift(cleaned);
//FitsIO.Write(cleaned, Path.Combine(outputFolder, "rec_CLEAN.fits"));
Tools.WriteToMeltCSV(PSF.Cut(cleaned), Path.Combine(outputFolder, "rec_CLEAN.csv"));
}
/// <summary>
/// Major cycle implementation for the Serial CD
/// </summary>
/// <param name="obsName"></param>
/// <param name="data"></param>
/// <param name="c"></param>
/// <param name="useGPU"></param>
/// <param name="psfCutFactor"></param>
/// <param name="maxMajorCycle"></param>
/// <param name="lambda"></param>
/// <param name="alpha"></param>
/// <param name="deconvIterations"></param>
/// <param name="deconvEpsilon"></param>
public static void ReconstructSerialCD(string obsName, MeasurementData data, GriddingConstants c, bool useGPU, int psfCutFactor, int maxMajorCycle, float lambda, float alpha, int deconvIterations, float deconvEpsilon)
{
var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies);
var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length];
for (int i = 0; i < data.Visibilities.GetLength(0); i++)
{
for (int j = 0; j < data.Visibilities.GetLength(1); j++)
{
for (int k = 0; k < data.Visibilities.GetLength(2); k++)
{
if (!data.Flags[i, j, k])
{
psfVis[i, j, k] = new Complex(1.0, 0);
}
else
{
psfVis[i, j, k] = new Complex(0, 0);
}
}
}
}
Console.WriteLine("gridding psf");
var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies);
var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var totalWatch = new Stopwatch();
var currentWatch = new Stopwatch();
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(psf, psfCutFactor);
var maxSidelobe = PSF.CalcMaxSidelobe(psf, psfCutFactor);
IDeconvolver deconvolver = null;
if (useGPU & GPUSerialCD.IsGPUSupported())
{
deconvolver = new GPUSerialCD(totalSize, psfCut, 1000);
}
else if (useGPU & !GPUSerialCD.IsGPUSupported())
{
Console.WriteLine("GPU not supported by library. Switching to CPU implementation");
deconvolver = new FastSerialCD(totalSize, psfCut);
}
else
{
deconvolver = new FastSerialCD(totalSize, psfCut);
}
var psfBMap = psfCut;
using (var gCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))))
using (var gCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))))
{
var currentGCalculator = gCalculator;
var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
var lambdaLipschitz = (float)(lambda * maxLipschitz);
var lambdaTrue = (float)(lambda * PSF.CalcMaxLipschitz(psf));
var switchedToOtherPsf = false;
var xImage = new float[c.GridSize, c.GridSize];
var residualVis = data.Visibilities;
DeconvolutionResult lastResult = null;
for (int cycle = 0; cycle < maxMajorCycle; cycle++)
{
Console.WriteLine("Beginning Major cycle " + cycle);
var dirtyGrid = IDG.GridW(c, metadata, residualVis, data.UVW, data.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, obsName + "_dirty_serial_majorCycle" + cycle + ".fits");
currentWatch.Restart();
totalWatch.Start();
var maxDirty = Residuals.GetMax(dirtyImage);
var gradients = gCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(gradients);
var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambdaLipschitz);
var objective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
var absMax = deconvolver.GetAbsMaxDiff(xImage, gradients, lambdaTrue, alpha);
if (absMax >= MAJOR_EPSILON)
{
lastResult = deconvolver.Deconvolve(xImage, gradients, currentLambda, alpha, deconvIterations, deconvEpsilon);
}
if (lambda == currentLambda & !switchedToOtherPsf)
{
currentGCalculator = gCalculator2;
lambda = lambdaTrue;
maxLipschitz = PSF.CalcMaxLipschitz(psf);
switchedToOtherPsf = true;
}
FitsIO.Write(xImage, obsName + "_model_serial_majorCycle" + cycle + ".fits");
currentWatch.Stop();
totalWatch.Stop();
if (absMax < MAJOR_EPSILON)
{
break;
}
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, data.UVW, data.Frequencies);
residualVis = Visibilities.Substract(data.Visibilities, modelVis, data.Flags);
}
Console.WriteLine("Reconstruction finished in (seconds): " + totalWatch.Elapsed.TotalSeconds);
}
}
public static void Run()
{
//var frequencies = FitsIO.ReadFrequencies(@"C:\Users\Jon\github\p9-data\small\fits\simulation_point\freq.fits");
//var uvw = FitsIO.ReadUVW(@"C:\Users\Jon\github\p9-data\small\fits\simulation_point\uvw.fits");
var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits");
var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits");
var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset
var visibilitiesCount = flags.Length;
int gridSize = 64;
int subgridsize = 16;
int kernelSize = 8;
int max_nr_timesteps = 64;
double cellSize = 2.0 / 3600.0 * PI / 180.0;
var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f);
var metadata = Partitioner.CreatePartition(c, uvw, frequencies);
var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies);
var psf = FFT.Backward(psfGrid, c.VisibilitiesCount);
FFT.Shift(psf);
var maxPsf = psf[gridSize / 2, gridSize / 2];
for (int i = 0; i < psf.GetLength(0); i++)
{
for (int j = 0; j < psf.GetLength(1); j++)
{
psf[i, j] = psf[i, j] / maxPsf;
}
}
FitsIO.Write(psf, "psf.fits");
var truth = new double[64, 64];
//truth[40, 50] = 1.5;
truth[0, 0] = 1.7;
var dirty = ConvolveFFTPadded(truth, psf);
FitsIO.Write(truth, "truth.fits");
FitsIO.Write(dirty, "dirty.fits");
var psf2 = ConvolveFFT(psf, psf);
var b = ConvolveFFTPadded(dirty, psf);
var a = psf2[gridSize / 2, gridSize / 2];
/*
* var integral = CalcPSf2Integral(psf);
* FitsIO.Write(integral, "psfIntegral.fits");
* var c0 = new double[64, 64];
* var qY = 0;
* var qX = 0;
* c0[qY, qX] = 1.0;
* c0 = Convolve(c0, psf);
* FitsIO.Write(c0, "cx0.fits");
* var cx = ConvolveFFT(c0, psf);
* FitsIO.Write(cx, "cx1.fits");
* var a2 = cx[qY, qX];
* var res = QueryIntegral(integral, qY, qX);*/
var x = new double[gridSize, gridSize];
//Deconv(x, dirty, psf, psf2, a, 0.0);
var dCopy = new double[gridSize, gridSize];
for (int i = 0; i < b.GetLength(0); i++)
{
for (int j = 0; j < b.GetLength(1); j++)
{
dCopy[i, j] = dirty[i, j];
}
}
var x2 = new double[gridSize, gridSize];
var converged = GreedyCD.Deconvolve2(x2, dirty, psf, 0.0, 1.0, 500, dCopy);
}
private static void ReconstructSerial(MeasurementData input, GriddingConstants c, float[,] fullPsf, string folder, string file, int processorCount)
{
var totalWatch = new Stopwatch();
var currentWatch = new Stopwatch();
var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize);
var psfCut = PSF.Cut(fullPsf, CUT_FACTOR_SERIAL);
var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, CUT_FACTOR_SERIAL);
var fastCD = new FastSerialCD(totalSize, psfCut, processorCount);
var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies);
var writer = new StreamWriter(folder + "/" + file + ".txt");
var psfBMap = psfCut;
using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))))
using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))))
{
var currentBMapCalculator = bMapCalculator;
var maxLipschitz = PSF.CalcMaxLipschitz(psfCut);
var lambda = (float)(LAMBDA * maxLipschitz);
var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf));
var alpha = ALPHA;
var switchedToOtherPsf = false;
var xImage = new float[c.GridSize, c.GridSize];
var residualVis = input.Visibilities;
DeconvolutionResult lastResult = null;
for (int cycle = 0; cycle < 6; cycle++)
{
Console.WriteLine("cycle " + cycle);
var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies);
var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount);
FFT.Shift(dirtyImage);
FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits");
currentWatch.Restart();
totalWatch.Start();
var maxDirty = Residuals.GetMax(dirtyImage);
var bMap = bMapCalculator.Convolve(dirtyImage);
var maxB = Residuals.GetMax(bMap);
var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f);
var currentSideLobe = maxB * maxSidelobe * correctionFactor;
var currentLambda = Math.Max(currentSideLobe / alpha, lambda);
var objective = Residuals.CalcPenalty(dirtyImage) + ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha);
var absMax = fastCD.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha);
if (absMax >= MAJOR_STOP)
{
lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, 1e-5f);
}
if (lambda == currentLambda & !switchedToOtherPsf)
{
currentBMapCalculator = bMapCalculator2;
lambda = lambdaTrue;
switchedToOtherPsf = true;
}
currentWatch.Stop();
totalWatch.Stop();
writer.WriteLine(cycle + ";" + currentLambda + ";" + objective + ";" + absMax + ";" + lastResult.IterationCount + ";" + totalWatch.Elapsed.TotalSeconds + ";" + currentWatch.Elapsed.TotalSeconds);
writer.Flush();
if (absMax < MAJOR_STOP)
{
break;
}
FFT.Shift(xImage);
var xGrid = FFT.Forward(xImage);
FFT.Shift(xImage);
var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies);
residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags);
}
}
}
