public static Data Load(string folder) { var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw.fits")); var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm); var visCount2 = 0; for (int i = 0; i < flags.GetLength(0); i++) { for (int j = 0; j < flags.GetLength(1); j++) { for (int k = 0; k < flags.GetLength(2); k++) { if (!flags[i, j, k]) { visCount2++; } } } } var visibilitiesCount = visCount2; var d = new Data(); d.frequencies = frequencies; d.visibilities = visibilities; d.uvw = uvw; d.flags = flags; d.visibilitiesCount = visibilitiesCount; return(d); }
private static DirtyImage ForwardCalculateB(Intracommunicator comm, GriddingConstants c, List <List <Subgrid> > metadata, Complex[,,] visibilities, double[,,] uvw, double[] frequencies, Complex[,] PsfCorrelation, float[,] psfCut, float maxSidelobe, Stopwatch watchIdg) { Stopwatch another = new Stopwatch(); comm.Barrier(); if (comm.Rank == 0) { watchIdg.Start(); } var localGrid = IDG.Grid(c, metadata, visibilities, uvw, frequencies); float[,] image = null; float maxSideLobeLevel = 0.0f; var grid_total = comm.Reduce <Complex[, ]>(localGrid, SequentialSum, 0); if (comm.Rank == 0) { var dirtyImage = FFT.BackwardFloat(grid_total, c.VisibilitiesCount); FFT.Shift(dirtyImage); if (comm.Rank == 0) { FitsIO.Write(dirtyImage, "dirtyImage.fits"); } maxSideLobeLevel = maxSidelobe * Residuals.GetMax(dirtyImage); //remove spheroidal image = Residuals.CalcGradientMap(dirtyImage, PsfCorrelation, new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); watchIdg.Stop(); } comm.Broadcast(ref maxSideLobeLevel, 0); comm.Broadcast(ref image, 0); return(new DirtyImage(image, maxSideLobeLevel)); }
public static bool Deconvolve(double[,] xImage, double[,] residuals, double[,] psf, double lambda, double alpha, int maxIteration = 100, double epsilon = 1e-4) { FitsIO.Write(residuals, "res.fits"); var yBlockSize = 2; var xBlockSize = 2; var psfCorrelated = CommonDeprecated.PSF.CalculateFourierCorrelation(psf, residuals.GetLength(0) - psf.GetLength(0), residuals.GetLength(1) - psf.GetLength(1)); var residualsFourier = FFT.Forward(residuals); residualsFourier = Common.Fourier2D.Multiply(residualsFourier, psfCorrelated); var bMap = FFT.Backward(residualsFourier, residualsFourier.Length); //FFT.Shift(bMap); FitsIO.Write(bMap, "bMap.fits"); var psf2Fourier = Common.Fourier2D.Multiply(psfCorrelated, psfCorrelated); var xDiff = new double[xImage.GetLength(0), xImage.GetLength(1)]; var blockInversion = CalcBlockInversion(psf, yBlockSize, xBlockSize); var random = new Random(123); var iter = 0; while (iter < maxIteration) { var yB = random.Next(xImage.GetLength(0) / yBlockSize); var xB = random.Next(xImage.GetLength(1) / xBlockSize); yB = 64 / yBlockSize; xB = 64 / xBlockSize; var block = CopyFrom(bMap, yB, xB, yBlockSize, xBlockSize); var optimized = block * blockInversion; var xOld = CopyFrom(xImage, yB, xB, yBlockSize, xBlockSize); optimized = xOld + optimized; //shrink /*for (int i = 0; i < optimized.Count; i++) * optimized[i] = Common.ShrinkElasticNet(optimized[i], lambda, alpha);*/ var optDiff = optimized - xOld; AddInto(xDiff, optDiff, yB, xB, yBlockSize, xBlockSize); AddInto(xImage, optDiff, yB, xB, yBlockSize, xBlockSize); FitsIO.Write(xImage, "xImageBlock.fits"); //update b-map var XDIFF = FFT.Forward(xDiff); XDIFF = Common.Fourier2D.Multiply(XDIFF, psf2Fourier); Common.Fourier2D.SubtractInPlace(residualsFourier, XDIFF); bMap = FFT.Backward(residualsFourier, residualsFourier.Length); FitsIO.Write(bMap, "bMap2.fits"); //clear from xDiff AddInto(xDiff, -optDiff, yB, xB, yBlockSize, xBlockSize); iter++; } return(false); }
public static void Run() { var folder = @"C:\dev\GitHub\p9-data\small\fits\simulation_point\"; var data = DataLoading.SimulatedPoints.Load(folder); var gridSizes = new int[] { 256, 512, 1024, 2048, 4096 }; Directory.CreateDirectory("GPUSpeedup"); var writer = new StreamWriter("GPUSpeedup/GPUSpeedup.txt", false); writer.WriteLine("imgSize;iterCPU;timeCPU;iterGPU;timeGPU"); foreach (var gridSize in gridSizes) { var visibilitiesCount = data.visibilitiesCount; int subgridsize = 8; int kernelSize = 4; int max_nr_timesteps = 1024; double cellSize = (1.0 * 256 / gridSize) / 3600.0 * Math.PI / 180.0; var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f); var metadata = Partitioner.CreatePartition(c, data.uvw, data.frequencies); var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw.fits")); var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm); var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies); var psf = FFT.BackwardFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var residualVis = data.visibilities; var dirtyGrid = IDG.Grid(c, metadata, residualVis, data.uvw, data.frequencies); var dirtyImage = FFT.BackwardFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); var totalSize = new Rectangle(0, 0, gridSize, gridSize); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psf, totalSize), new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var bMapCPU = bMapCalculator.Convolve(dirtyImage); var bMapGPU = bMapCalculator.Convolve(dirtyImage); var fastCD = new FastSerialCD(totalSize, psf); var gpuCD = new GPUSerialCD(totalSize, psf, 1000); var lambda = 0.5f * fastCD.MaxLipschitz; var alpha = 0.5f; var xCPU = new float[gridSize, gridSize]; var cpuResult = fastCD.Deconvolve(xCPU, bMapCPU, lambda, alpha, 10000, 1e-8f); FitsIO.Write(xCPU, "GPUSpeedup/cpuResult" + gridSize + ".fits"); var xGPU = new float[gridSize, gridSize]; var gpuResult = gpuCD.Deconvolve(xGPU, bMapGPU, lambda, alpha, 10000, 1e-8f); FitsIO.Write(xCPU, "GPUSpeedup/gpuResult" + gridSize + ".fits"); writer.WriteLine(gridSize + ";" + cpuResult.IterationCount + ";" + cpuResult.ElapsedTime.TotalSeconds + ";" + gpuResult.IterationCount + ";" + gpuResult.ElapsedTime.TotalSeconds); writer.Flush(); } writer.Close(); }
private static void ReconstructRandom(MeasurementData input, GriddingConstants c, float[,] psf, int blockSize, int iterCount, string file) { var cutFactor = 8; var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(psf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(psf, cutFactor); var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut)); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(psf)); var alpha = ALPHA; ApproxFast.LAMBDA_TEST = lambdaTrue; ApproxFast.ALPHA_TEST = alpha; var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies); var random = new Random(123); var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.0f, 0.0f, false, true, false); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); var data = new ApproxFast.TestingData(new StreamWriter(file + "_tmp.txt")); var xImage = new float[c.GridSize, c.GridSize]; var xCorr = Copy(xImage); var residualVis = input.Visibilities; var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); var gCorr = new float[c.GridSize, c.GridSize]; var shared = new ApproxFast.SharedData(currentLambda, alpha, 1, 1, 8, CountNonZero(psfCut), approx.psf2, approx.aMap, xImage, xCorr, bMap, gCorr, new Random()); shared.ActiveSet = ApproxFast.GetActiveSet(xImage, bMap, shared.YBlockSize, shared.XBlockSize, lambda, alpha, shared.AMap); shared.BlockLock = new int[shared.ActiveSet.Count]; shared.maxLipschitz = (float)PSF.CalcMaxLipschitz(psfCut); shared.MaxConcurrentIterations = 1000; approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, 1, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf); var output = Tools.LMC.CutN132Remnant(xImage); Tools.WriteToMeltCSV(output.Item1, file + "_1k.csv", output.Item2, output.Item3); FitsIO.Write(output.Item1, file + "_1k.fits"); FitsIO.Write(xImage, file + "_1k2.fits"); approx.DeconvolveConcurrentTest(data, 0, 0, 0.0, shared, iterCount, 1e-5f, Copy(xImage), dirtyImage, psfCut, psf); output = Tools.LMC.CutN132Remnant(xImage); Tools.WriteToMeltCSV(output.Item1, file + "_10k.csv", output.Item2, output.Item3); FitsIO.Write(output.Item1, file + "_10k.fits"); FitsIO.Write(xImage, file + "_10k2.fits"); }
public static LocalDataset LoadSubsetTinyMeerKAT(string folder) { var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw0.fits")); var flags = FitsIO.ReadFlags(Path.Combine(folder, "flags0.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length); var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis0.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, 2.0); return(new LocalDataset(frequencies, uvw, flags, visibilities)); }
public static MeasurementData LoadSimulatedPoints(string folder) { var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw.fits")); var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm); return(new MeasurementData(visibilities, uvw, frequencies, flags)); }
public static LocalDataset LoadSimulated(string folder) { var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw.fits")); var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, 2.0); return(new LocalDataset(frequencies, uvw, flags, visibilities)); }
public static Data Load(string folder) { var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw0.fits")); var flags = FitsIO.ReadFlags(Path.Combine(folder, "flags0.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length); double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis0.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm); for (int i = 1; i < 8; i++) { var uvw0 = FitsIO.ReadUVW(Path.Combine(folder, "uvw" + i + ".fits")); var flags0 = FitsIO.ReadFlags(Path.Combine(folder, "flags" + i + ".fits"), uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length); var visibilities0 = FitsIO.ReadVisibilities(Path.Combine(folder, "vis" + i + ".fits"), uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, norm); uvw = FitsIO.Stitch(uvw, uvw0); flags = FitsIO.Stitch(flags, flags0); visibilities = FitsIO.Stitch(visibilities, visibilities0); } var visCount2 = 0; for (int i = 0; i < flags.GetLength(0); i++) { for (int j = 0; j < flags.GetLength(1); j++) { for (int k = 0; k < flags.GetLength(2); k++) { if (!flags[i, j, k]) { visCount2++; } } } } var visibilitiesCount = visCount2; var d = new Data(); d.frequencies = frequencies; d.visibilities = visibilities; d.uvw = uvw; d.flags = flags; d.visibilitiesCount = visibilitiesCount; return(d); }
public static void Generate(string outputFolder) { Directory.CreateDirectory(outputFolder); var reference = FitsIO.ReadImage(Path.Combine(INPUT_DIR, "xReference4.fits")); Tools.WriteToMeltCSV(reference, Path.Combine(outputFolder, "CD-reference.csv")); var n132 = Tools.LMC.CutN132Remnant(reference); Tools.WriteToMeltCSV(n132.Item1, Path.Combine(outputFolder, "CD-N132.csv"), n132.Item2, n132.Item3); var calibration = Tools.LMC.CutCalibration(reference); Tools.WriteToMeltCSV(calibration.Item1, Path.Combine(outputFolder, "CD-Calibration.csv"), calibration.Item2, calibration.Item3); var residual = FitsIO.ReadCASAFits(Path.Combine(INPUT_DIR, "cd-residual.fits")); Tools.WriteToMeltCSV(residual, Path.Combine(outputFolder, "CD-reference-residuals.csv")); var dirtyImage = FitsIO.ReadImage(Path.Combine(INPUT_DIR, "dirty0.fits")); Tools.WriteToMeltCSV(dirtyImage, Path.Combine(outputFolder, "CD-dirty.csv")); var reconstruction = FitsIO.ReadCASAFits(Path.Combine(INPUT_DIR, "cd-image.fits")); Tools.WriteToMeltCSV(reconstruction, Path.Combine(outputFolder, "CD-restored.csv")); var image = new float[reconstruction.GetLength(0), reconstruction.GetLength(1)]; for (int i = 0; i < reconstruction.GetLength(0); i++) { for (int j = 0; j < reconstruction.GetLength(1); j++) { image[i, j] = reconstruction[i, j] - residual[i, j]; } } Tools.WriteToMeltCSV(image, Path.Combine(outputFolder, "CD-image-no-residuals.csv")); var example = Tools.LMC.CutExampleImage(image); Tools.WriteToMeltCSV(example.Item1, Path.Combine(outputFolder, "CD-example.csv"), example.Item2, example.Item3); n132 = Tools.LMC.CutN132Remnant(image); Tools.WriteToMeltCSV(n132.Item1, Path.Combine(outputFolder, "CD-image-N132.csv"), n132.Item2, n132.Item3); calibration = Tools.LMC.CutCalibration(image); Tools.WriteToMeltCSV(calibration.Item1, Path.Combine(outputFolder, "CD-image-Calibration.csv"), calibration.Item2, calibration.Item3); }
public static List <Tuple <int, int> > GetActiveSet(float[,] xExplore, float[,] gExplore, int yBlockSize, int xBlockSize, float lambda, float alpha, float[,] lipschitzMap) { var debug = new float[xExplore.GetLength(0), xExplore.GetLength(1)]; var output = new List <Tuple <int, int> >(); for (int i = 0; i < xExplore.GetLength(0) / yBlockSize; i++) { for (int j = 0; j < xExplore.GetLength(1) / xBlockSize; j++) { var yPixel = i * yBlockSize; var xPixel = j * xBlockSize; var nonZero = false; for (int y = yPixel; y < yPixel + yBlockSize; y++) { for (int x = xPixel; x < xPixel + xBlockSize; x++) { var lipschitz = lipschitzMap[y, x]; var tmp = gExplore[y, x] + xExplore[y, x] * lipschitz; tmp = ElasticNet.ProximalOperator(tmp, lipschitz, lambda, alpha); if (0.0f < Math.Abs(tmp - xExplore[y, x])) { nonZero = true; } } } if (nonZero) { output.Add(new Tuple <int, int>(i, j)); for (int y = yPixel; y < yPixel + yBlockSize; y++) { for (int x = xPixel; x < xPixel + xBlockSize; x++) { debug[y, x] = 1.0f; } } } } } FitsIO.Write(debug, "activeSet.fits"); return(output); }
private List <Tuple <int, int> > GetActiveSet(float[,] xExplore, float[,] gExplore, float lambda, float alpha, float lipschitz) { var debug = new float[xExplore.GetLength(0), xExplore.GetLength(1)]; var output = new List <Tuple <int, int> >(); for (int i = 0; i < xExplore.GetLength(0) / yBlockSize; i++) { for (int j = 0; j < xExplore.GetLength(1) / xBlockSize; j++) { var yPixel = i * yBlockSize; var xPixel = j * xBlockSize; var nonZero = false; for (int y = yPixel; y < yPixel + yBlockSize; y++) { for (int x = xPixel; x < xPixel + xBlockSize; x++) { var tmp = gExplore[y, x] + xExplore[y, x] * lipschitz; tmp = ElasticNet.ProximalOperator(tmp, lipschitz, lambda, alpha); if (ACTIVE_SET_CUTOFF < Math.Abs(tmp - xExplore[y, x])) { nonZero = true; } } } if (nonZero) { output.Add(new Tuple <int, int>(i, j)); for (int y = yPixel; y < yPixel + yBlockSize; y++) { for (int x = xPixel; x < xPixel + xBlockSize; x++) { debug[y, x] = 1.0f; } } } } } FitsIO.Write(debug, "activeSet.fits"); //can write max change for convergence purposes return(output); }
public static MeasurementData LoadLMC(string folder) { var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw0.fits")); var flags = FitsIO.ReadFlags(Path.Combine(folder, "flags0.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length); double norm = 2.0; var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis0.fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, norm); for (int i = 1; i < 8; i++) { var uvw0 = FitsIO.ReadUVW(Path.Combine(folder, "uvw" + i + ".fits")); var flags0 = FitsIO.ReadFlags(Path.Combine(folder, "flags" + i + ".fits"), uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length); var visibilities0 = FitsIO.ReadVisibilities(Path.Combine(folder, "vis" + i + ".fits"), uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, norm); uvw = FitsIO.Stitch(uvw, uvw0); flags = FitsIO.Stitch(flags, flags0); visibilities = FitsIO.Stitch(visibilities, visibilities0); } return(new MeasurementData(visibilities, uvw, frequencies, flags)); }
private static void GeneratePCDMComparison(string outputFolder) { Directory.CreateDirectory(outputFolder); var serial = FitsIO.ReadImage(Path.Combine(INPUT_DIR, "xImage_serial_4.fits")); var n132 = Tools.LMC.CutN132Remnant(serial); Tools.WriteToMeltCSV(n132.Item1, Path.Combine(outputFolder, "SerialCD-N132.csv"), n132.Item2, n132.Item3); var calibration = Tools.LMC.CutCalibration(serial); Tools.WriteToMeltCSV(calibration.Item1, Path.Combine(outputFolder, "SerialCD-Calibration.csv"), calibration.Item2, calibration.Item3); var pcdm = FitsIO.ReadImage(Path.Combine(INPUT_DIR, "xImage_pcdm_5.fits")); var n132_2 = Tools.LMC.CutN132Remnant(pcdm); Tools.WriteToMeltCSV(n132_2.Item1, Path.Combine(outputFolder, "PCDM-N132.csv"), n132.Item2, n132.Item3); var calibration2 = Tools.LMC.CutCalibration(pcdm); Tools.WriteToMeltCSV(calibration2.Item1, Path.Combine(outputFolder, "PCDM-Calibration.csv"), calibration.Item2, calibration.Item3); }
public static LocalDataset LoadTinyMeerKAT(Intracommunicator comm, string folder) { var beginIdx = comm.Rank * 8 / comm.Size; var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw" + comm.Rank + ".fits")); var flags = FitsIO.ReadFlags(Path.Combine(folder, "flags" + comm.Rank + ".fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length); var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis" + comm.Rank + ".fits"), uvw.GetLength(0), uvw.GetLength(1), frequencies.Length, 2.0); for (int i = beginIdx + 1; i < beginIdx + 8 / comm.Size; i++) { var uvw0 = FitsIO.ReadUVW(Path.Combine(folder, "uvw" + i + ".fits")); var flags0 = FitsIO.ReadFlags(Path.Combine(folder, "flags" + i + ".fits"), uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length); var visibilities0 = FitsIO.ReadVisibilities(Path.Combine(folder, "vis" + i + ".fits"), uvw0.GetLength(0), uvw0.GetLength(1), frequencies.Length, 2.0); uvw = FitsIO.Stitch(uvw, uvw0); flags = FitsIO.Stitch(flags, flags0); visibilities = FitsIO.Stitch(visibilities, visibilities0); } return(new LocalDataset(frequencies, uvw, flags, visibilities)); }
public static void RunSimulated(string[] args) { using (var env = new MPI.Environment(ref args, MPI.Threading.Serialized)) { var proc = Process.GetCurrentProcess(); var name = proc.ProcessName; Console.WriteLine(" name: " + name); //System.Threading.Thread.Sleep(17000); var comm = Communicator.world; //READ DATA var folder = @"C:\dev\GitHub\p9-data\small\fits\simulation_point\"; var bla = @"C:\dev\GitHub\p9-data\small\fits\simulation_point"; var fullData = DistributedData.LoadSimulated(folder); var data = DistributedData.SplitDataAmongNodes(comm, fullData); var totalVisibilitiesCount = fullData.VisibilitiesCount; int gridSize = 256; int subgridsize = 16; int kernelSize = 8; int max_nr_timesteps = 512; double cellSize = 1.0 / 3600.0 * PI / 180.0; var c = new GriddingConstants(totalVisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f); comm.Barrier(); if (comm.Rank == 0) { Console.WriteLine("Done Reading, Starting reconstruction"); } var reconstruction = MPIMajorCycle.Reconstruct(comm, data, c, 1, 0.5f, 0.8f, 1000); if (comm.Rank == 0) { FitsIO.Write(reconstruction, "simulatedReconstruction.fits"); } } }
public static void Generate(string outputFolder) { Directory.CreateDirectory(outputFolder); var model = FitsIO.ReadCASAFits(Path.Combine(INPUT_DIR, "nat-iuwt-model.fits")); var residual = FitsIO.ReadCASAFits(Path.Combine(INPUT_DIR, "nat-iuwt-residual.fits")); Tools.WriteToMeltCSV(residual, Path.Combine(outputFolder, "iuwt-residuals.csv")); Tools.WriteToMeltCSV(model, Path.Combine(outputFolder, "iuwt-model.csv")); var reconstruction = FitsIO.ReadCASAFits(Path.Combine(INPUT_DIR, "nat-iuwt-image.fits")); var image = new float[reconstruction.GetLength(0), reconstruction.GetLength(1)]; for (int i = 0; i < reconstruction.GetLength(0); i++) { for (int j = 0; j < reconstruction.GetLength(1); j++) { image[i, j] = reconstruction[i, j] - residual[i, j]; } } var example = Tools.LMC.CutExampleImage(image); Tools.WriteToMeltCSV(example.Item1, Path.Combine(outputFolder, "iuwt-example.csv"), example.Item2, example.Item3); var n132 = Tools.LMC.CutN132Remnant(image); Tools.WriteToMeltCSV(n132.Item1, Path.Combine(outputFolder, "iuwt-image-N132.csv"), n132.Item2, n132.Item3); var calibration = Tools.LMC.CutCalibration(image); Tools.WriteToMeltCSV(calibration.Item1, Path.Combine(outputFolder, "iuwt-image-Calibration.csv"), calibration.Item2, calibration.Item3); n132 = Tools.LMC.CutN132Remnant(model); Tools.WriteToMeltCSV(n132.Item1, Path.Combine(outputFolder, "iuwt-N132.csv"), n132.Item2, n132.Item3); calibration = Tools.LMC.CutCalibration(model); Tools.WriteToMeltCSV(calibration.Item1, Path.Combine(outputFolder, "iuwt-Calibration.csv"), calibration.Item2, calibration.Item3); }
public static void RunTinyMeerKAT(string[] args) { using (var env = new MPI.Environment(ref args, MPI.Threading.Serialized)) { var proc = Process.GetCurrentProcess(); var name = proc.ProcessName; Console.WriteLine(" name: " + name); var comm = Communicator.world; var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = DistributedData.LoadTinyMeerKAT2(comm.Rank, comm.Size, folder); var totalVisCount = comm.Allreduce(data.VisibilitiesCount, (x, y) => x + y); int gridSize = 3072; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 1.5 / 3600.0 * PI / 180.0; var c = new GriddingConstants(totalVisCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f); comm.Barrier(); if (comm.Rank == 0) { Console.WriteLine("Done Reading, Starting reconstruction"); } var lambda = 0.4f; var alpha = 0.1f; var reconstruction = MPIMajorCycle.Reconstruct(comm, data, c, 2, lambda, alpha, 10000); if (comm.Rank == 0) { FitsIO.Write(reconstruction, "tinyMeerKATReconstruction.fits"); } } }
public static void Deconv(double[,] xImage, double[,] dirty, double[,] psf, double[,] psf2, double a, double lambda, int maxIter = 1) { var aMap = CalcPSf2IntegralSquared(psf); var aMapInverted = CalcPSf2IntegralSquared2(psf); var iter = 0; var converged = false; var FO = new double[xImage.GetLength(0), xImage.GetLength(1)]; var XO = new double[xImage.GetLength(0), xImage.GetLength(1)]; while (iter < maxIter & !converged) { var convolved = ConvolveFFTPadded(xImage, psf); var residuals = Subtract(dirty, convolved); FitsIO.Write(residuals, "deconv_residuals_" + iter + ".fits"); var bMap = ConvolveFFTPaddedInverted(residuals, psf); FitsIO.Write(bMap, "deconv_bMap_" + iter + ".fits"); var objectiveVal = CalcDataObjective(residuals); objectiveVal += CalcL1Objective(xImage, aMap, lambda); var minVal = Double.MaxValue; var yPixel = -1; var xPixel = -1; var xNew = 0.0; for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { var currentB = bMap[i, j]; var currentA = QueryIntegral(aMap, i, j); //var xDiff = currentB / a; var xDiff = currentB / currentA; var x = xImage[i, j] + xDiff; x = ShrinkAbsolute(x, lambda); x = Math.Max(x, 0); XO[i, j] = x; var currentO = EstimateObjective(xImage, dirty, psf, i, j, x, aMap, lambda); if (Math.Abs(x - xImage[i, j]) > 1e-6) { if (currentO <= objectiveVal + 1e-6) { Console.Write(""); } else { Console.Write("ERROR"); } } FO[i, j] = currentO; if (minVal > currentO) { minVal = currentO; xNew = x; yPixel = i; xPixel = j; } } } FitsIO.Write(FO, "deconv_FO_" + iter + ".fits"); FitsIO.Write(XO, "deconv_XO_" + iter + ".fits"); var old = xImage[yPixel, xPixel]; if (Math.Abs(old - xNew) > 1e-2) { xImage[yPixel, xPixel] = xNew; } else { converged = true; } iter++; FitsIO.Write(xImage, "deconv_x_" + iter + ".fits"); } }
public static void Run() { //var frequencies = FitsIO.ReadFrequencies(@"C:\Users\Jon\github\p9-data\small\fits\simulation_point\freq.fits"); //var uvw = FitsIO.ReadUVW(@"C:\Users\Jon\github\p9-data\small\fits\simulation_point\uvw.fits"); var frequencies = FitsIO.ReadFrequencies(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\freq.fits"); var uvw = FitsIO.ReadUVW(@"C:\dev\GitHub\p9-data\small\fits\simulation_point\uvw.fits"); var flags = new bool[uvw.GetLength(0), uvw.GetLength(1), frequencies.Length]; //completely unflagged dataset var visibilitiesCount = flags.Length; int gridSize = 64; int subgridsize = 16; int kernelSize = 8; int max_nr_timesteps = 64; double cellSize = 2.0 / 3600.0 * PI / 180.0; var c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, 1, 0.0f); var metadata = Partitioner.CreatePartition(c, uvw, frequencies); var psfGrid = IDG.GridPSF(c, metadata, uvw, flags, frequencies); var psf = FFT.Backward(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var maxPsf = psf[gridSize / 2, gridSize / 2]; for (int i = 0; i < psf.GetLength(0); i++) { for (int j = 0; j < psf.GetLength(1); j++) { psf[i, j] = psf[i, j] / maxPsf; } } FitsIO.Write(psf, "psf.fits"); var truth = new double[64, 64]; //truth[40, 50] = 1.5; truth[0, 0] = 1.7; var dirty = ConvolveFFTPadded(truth, psf); FitsIO.Write(truth, "truth.fits"); FitsIO.Write(dirty, "dirty.fits"); var psf2 = ConvolveFFT(psf, psf); var b = ConvolveFFTPadded(dirty, psf); var a = psf2[gridSize / 2, gridSize / 2]; /* * var integral = CalcPSf2Integral(psf); * FitsIO.Write(integral, "psfIntegral.fits"); * var c0 = new double[64, 64]; * var qY = 0; * var qX = 0; * c0[qY, qX] = 1.0; * c0 = Convolve(c0, psf); * FitsIO.Write(c0, "cx0.fits"); * var cx = ConvolveFFT(c0, psf); * FitsIO.Write(cx, "cx1.fits"); * var a2 = cx[qY, qX]; * var res = QueryIntegral(integral, qY, qX);*/ var x = new double[gridSize, gridSize]; //Deconv(x, dirty, psf, psf2, a, 0.0); var dCopy = new double[gridSize, gridSize]; for (int i = 0; i < b.GetLength(0); i++) { for (int j = 0; j < b.GetLength(1); j++) { dCopy[i, j] = dirty[i, j]; } } var x2 = new double[gridSize, gridSize]; var converged = GreedyCD.Deconvolve2(x2, dirty, psf, 0.0, 1.0, 500, dCopy); }
private static ReconstructionInfo ReconstructGradientApprox(Data input, float[,] fullPsf, string folder, int cutFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon) { var info = new ReconstructionInfo(); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var psfBMap = psfCut; var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))); var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))); var fastCD = new FastSerialCD(totalSize, psfCut); var fastCD2 = new FastSerialCD(totalSize, psfCut); fastCD2.ResetLipschitzMap(fullPsf); FitsIO.Write(psfCut, folder + cutFactor + "psf.fits"); var lambda = LAMBDA_GLOBAL * fastCD.MaxLipschitz; var lambdaTrue = (float)(LAMBDA_GLOBAL * PSF.CalcMaxLipschitz(fullPsf)); var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; DeconvolutionResult lastResult = null; var firstTimeConverged = false; var lastLambda = 0.0f; for (int cycle = 0; cycle < maxMajor; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits"); //calc data and reg penalty var dataPenalty = Residuals.CalcPenalty(dirtyImage); var regPenalty = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha); info.lastDataPenalty = dataPenalty; info.lastRegPenalty = regPenalty; var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + ".fits"); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = Math.Max(currentSideLobe / alpha, lambda); writer.Write(cycle + ";" + currentLambda + ";" + currentSideLobe + ";" + ";" + fastCD2.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";");; writer.Flush(); //check wether we can minimize the objective further with the current psf var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff; var minimumReached = (lastResult != null && lastResult.Converged && fastCD2.GetAbsMaxDiff(xImage, dirtyImage, lambdaTrue, alpha) < MAJOR_EPSILON && currentLambda == lambda); if (lambda == lastLambda & !firstTimeConverged) { firstTimeConverged = true; minimumReached = false; } if (!objectiveReached & !minimumReached) { //writer.Write(firstTimeConverged + ";"); //writer.Flush(); info.totalDeconv.Start(); if (!firstTimeConverged) { lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon); } else { bMap = bMapCalculator2.Convolve(dirtyImage); //FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + "_full.fits"); maxB = Residuals.GetMax(bMap); correctionFactor = Math.Max(maxB / (maxDirty * fastCD2.MaxLipschitz), 1.0f); currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = Math.Max(currentSideLobe / alpha, lambdaTrue); info.totalDeconv.Start(); lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon); info.totalDeconv.Stop(); } info.totalDeconv.Stop(); FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits"); writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n"); writer.Flush(); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } else { writer.Write(false + ";0;0\n"); writer.Flush(); break; } lastLambda = currentLambda; } bMapCalculator.Dispose(); bMapCalculator2.Dispose(); return(info); }
public static void RunSpeedLarge() { var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = LMC.Load(folder); int gridSize = 3072; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 1.5 / 3600.0 * PI / 180.0; int wLayerCount = 24; var maxW = 0.0; for (int i = 0; i < data.uvw.GetLength(0); i++) { for (int j = 0; j < data.uvw.GetLength(1); j++) { maxW = Math.Max(maxW, Math.Abs(data.uvw[i, j, 2])); } } maxW = Partitioner.MetersToLambda(maxW, data.frequencies[data.frequencies.Length - 1]); var visCount2 = 0; for (int i = 0; i < data.flags.GetLength(0); i++) { for (int j = 0; j < data.flags.GetLength(1); j++) { for (int k = 0; k < data.flags.GetLength(2); k++) { if (!data.flags[i, j, k]) { visCount2++; } } } } double wStep = maxW / (wLayerCount); data.c = new GriddingConstants(data.visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep); data.metadata = Partitioner.CreatePartition(data.c, data.uvw, data.frequencies); var psfVis = new Complex[data.uvw.GetLength(0), data.uvw.GetLength(1), data.frequencies.Length]; for (int i = 0; i < data.visibilities.GetLength(0); i++) { for (int j = 0; j < data.visibilities.GetLength(1); j++) { for (int k = 0; k < data.visibilities.GetLength(2); k++) { if (!data.flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(data.c, data.metadata, psfVis, data.uvw, data.frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, data.c.VisibilitiesCount); FFT.Shift(psf); Directory.CreateDirectory("PSFSpeedExperimentApproxDeconv"); FitsIO.Write(psf, "psfFull.fits"); //tryout with simply cutting the PSF ReconstructionInfo experimentInfo = null; var psfCuts = new int[] { 2, 8, 16, 32, 64 }; var outFolder = "PSFSpeedExperimentApproxDeconv"; outFolder += @"\"; var fileHeader = "cycle;lambda;sidelobe;maxPixel;dataPenalty;regPenalty;currentRegPenalty;converged;iterCount;ElapsedTime"; /* * foreach (var cut in psfCuts) * { * using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) * { * writer.WriteLine(fileHeader); * experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 8, cut+"dirty", cut+"x", writer, 0.0, 1e-5f, false); * File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); * } * }*/ Directory.CreateDirectory("PSFSpeedExperimentApproxPSF"); outFolder = "PSFSpeedExperimentApproxPSF"; outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 8, cut + "dirty", cut + "x", writer, 0.0, 1e-5f, true); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } }
public static float[,] Reconstruct(Intracommunicator comm, DistributedData.LocalDataset local, GriddingConstants c, int maxCycle, float lambda, float alpha, int iterPerCycle = 1000, bool usePathDeconvolution = false) { var watchTotal = new Stopwatch(); var watchForward = new Stopwatch(); var watchBackward = new Stopwatch(); var watchDeconv = new Stopwatch(); watchTotal.Start(); var metadata = Partitioner.CreatePartition(c, local.UVW, local.Frequencies); var patchSize = CalculateLocalImageSection(comm.Rank, comm.Size, c.GridSize, c.GridSize); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); //calculate psf and prepare for correlation in the Fourier space var psf = CalculatePSF(comm, c, metadata, local.UVW, local.Flags, local.Frequencies); Complex[,] PsfCorrelation = null; var maxSidelobe = PSF.CalcMaxSidelobe(psf); lambda = (float)(lambda * PSF.CalcMaxLipschitz(psf)); StreamWriter writer = null; if (comm.Rank == 0) { FitsIO.Write(psf, "psf.fits"); Console.WriteLine("done PSF gridding "); PsfCorrelation = PSF.CalcPaddedFourierCorrelation(psf, totalSize); writer = new StreamWriter(comm.Size + "runtimestats.txt"); } var deconvovler = new MPIGreedyCD(comm, totalSize, patchSize, psf); var residualVis = local.Visibilities; var xLocal = new float[patchSize.YEnd - patchSize.Y, patchSize.XEnd - patchSize.X]; for (int cycle = 0; cycle < maxCycle; cycle++) { if (comm.Rank == 0) { Console.WriteLine("cycle " + cycle); } var dirtyImage = ForwardCalculateB(comm, c, metadata, residualVis, local.UVW, local.Frequencies, PsfCorrelation, psf, maxSidelobe, watchForward); var bLocal = GetImgSection(dirtyImage.Image, patchSize); MPIGreedyCD.Statistics lastRun; if (usePathDeconvolution) { var currentLambda = Math.Max(1.0f / alpha * dirtyImage.MaxSidelobeLevel, lambda); lastRun = deconvovler.DeconvolvePath(xLocal, bLocal, currentLambda, 4.0f, alpha, 5, iterPerCycle, 2e-5f); } else { lastRun = deconvovler.Deconvolve(xLocal, bLocal, lambda, alpha, iterPerCycle, 1e-5f); } if (comm.Rank == 0) { WriteToFile(cycle, lastRun, writer); if (lastRun.Converged) { Console.WriteLine("-----------------------------CONVERGED!!!!------------------------"); } else { Console.WriteLine("-------------------------------not converged----------------------"); } } comm.Barrier(); if (comm.Rank == 0) { watchDeconv.Stop(); } float[][,] totalX = null; comm.Gather(xLocal, 0, ref totalX); Complex[,] modelGrid = null; if (comm.Rank == 0) { watchBackward.Start(); var x = new float[c.GridSize, c.GridSize]; StitchImage(totalX, x, comm.Size); FitsIO.Write(x, "xImage_" + cycle + ".fits"); FFT.Shift(x); modelGrid = FFT.Forward(x); } comm.Broadcast(ref modelGrid, 0); var modelVis = IDG.DeGrid(c, metadata, modelGrid, local.UVW, local.Frequencies); residualVis = Visibilities.Substract(local.Visibilities, modelVis, local.Flags); } writer.Close(); float[][,] gatherX = null; comm.Gather(xLocal, 0, ref gatherX); float[,] reconstructed = null; if (comm.Rank == 0) { reconstructed = new float[c.GridSize, c.GridSize];; StitchImage(gatherX, reconstructed, comm.Size); } return(reconstructed); }
private static void Reconstruct(Data input, int cutFactor, float[,] fullPsf, string folder, string file, int threads, int blockSize, bool accelerated, float randomPercent, float searchPercent) { var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1))); var random = new Random(123); var approx = new ApproxFast(totalSize, psfCut, threads, blockSize, randomPercent, searchPercent, false, true); var maxLipschitzCut = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * PSF.CalcMaxLipschitz(psfCut)); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf)); var alpha = ALPHA; ApproxFast.LAMBDA_TEST = lambdaTrue; ApproxFast.ALPHA_TEST = alpha; var switchedToOtherPsf = false; var writer = new StreamWriter(folder + "/" + file + "_lambda.txt"); var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt")); var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; for (int cycle = 0; cycle < 7; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits"); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitzCut), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); writer.WriteLine("cycle" + ";" + currentLambda); writer.Flush(); approx.DeconvolveTest(data, cycle, 0, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f); FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits"); if (currentLambda == lambda & !switchedToOtherPsf) { approx.ResetAMap(fullPsf); lambda = lambdaTrue; switchedToOtherPsf = true; writer.WriteLine("switched"); writer.Flush(); } FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } writer.Close(); }
public static void RunApproximationMethods() { var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = LMC.Load(folder); var rootFolder = Directory.GetCurrentDirectory(); var maxW = 0.0; for (int i = 0; i < data.uvw.GetLength(0); i++) { for (int j = 0; j < data.uvw.GetLength(1); j++) { maxW = Math.Max(maxW, Math.Abs(data.uvw[i, j, 2])); } } maxW = Partitioner.MetersToLambda(maxW, data.frequencies[data.frequencies.Length - 1]); var visCount2 = 0; for (int i = 0; i < data.flags.GetLength(0); i++) { for (int j = 0; j < data.flags.GetLength(1); j++) { for (int k = 0; k < data.flags.GetLength(2); k++) { if (!data.flags[i, j, k]) { visCount2++; } } } } var visibilitiesCount = visCount2; int gridSize = 3072; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 1.5 / 3600.0 * PI / 180.0; int wLayerCount = 24; double wStep = maxW / (wLayerCount); data.c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep); data.metadata = Partitioner.CreatePartition(data.c, data.uvw, data.frequencies); data.visibilitiesCount = visibilitiesCount; var psfVis = new Complex[data.uvw.GetLength(0), data.uvw.GetLength(1), data.frequencies.Length]; for (int i = 0; i < data.visibilities.GetLength(0); i++) { for (int j = 0; j < data.visibilities.GetLength(1); j++) { for (int k = 0; k < data.visibilities.GetLength(2); k++) { if (!data.flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(data.c, data.metadata, psfVis, data.uvw, data.frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, data.c.VisibilitiesCount); FFT.Shift(psf); Directory.CreateDirectory("PSFSizeExperimentLarge"); Directory.SetCurrentDirectory("PSFSizeExperimentLarge"); FitsIO.Write(psf, "psfFull.fits"); Console.WriteLine(PSF.CalcMaxLipschitz(psf)); Console.WriteLine(visCount2); //reconstruct with full psf and find reference objective value var fileHeader = "cycle;lambda;sidelobe;maxPixel;dataPenalty;regPenalty;currentRegPenalty;converged;iterCount;ElapsedTime"; var objectiveCutoff = REFERENCE_L2_PENALTY + REFERENCE_ELASTIC_PENALTY; var recalculateFullPSF = true; if (recalculateFullPSF) { ReconstructionInfo referenceInfo = null; using (var writer = new StreamWriter("1Psf.txt", false)) { writer.WriteLine(fileHeader); referenceInfo = ReconstructSimple(data, psf, "", 1, 12, "dirtyReference", "xReference", writer, 0.0, 1e-5f, false); File.WriteAllText("1PsfTotal.txt", referenceInfo.totalDeconv.Elapsed.ToString()); } objectiveCutoff = referenceInfo.lastDataPenalty + referenceInfo.lastRegPenalty; } //tryout with simply cutting the PSF ReconstructionInfo experimentInfo = null; var psfCuts = new int[] { 16, 32 }; var outFolder = "cutPsf"; Directory.CreateDirectory(outFolder); outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 12, cut + "dirty", cut + "x", writer, 0.0, 1e-5f, false); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } //Tryout with cutting the PSF, but starting from the true bMap outFolder = "cutPsf2"; Directory.CreateDirectory(outFolder); outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 12, cut + "dirty", cut + "x", writer, 0.0, 1e-5f, true); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } //combined, final solution. Cut the psf in half, optimize until convergence, and then do one more major cycle with the second method outFolder = "properSolution"; Directory.CreateDirectory(outFolder); outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructGradientApprox(data, psf, outFolder, cut, 12, cut + "dirty", cut + "x", writer, 0.0, 1e-5f); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } Directory.SetCurrentDirectory(rootFolder); }
public static bool Deconvolve(double[,] xImage, double[,] residuals, double[,] psf, double lambda, double alpha, int maxIteration = 100, double epsilon = 1e-4) { FitsIO.Write(residuals, "res.fits"); var yBlockSize = 2; var xBlockSize = 2; var PSFCorrelated = CommonDeprecated.PSF.CalculateFourierCorrelation(psf, psf.GetLength(0), psf.GetLength(1)); var RES = FFT.Forward(residuals); var BMAPFourier = Common.Fourier2D.Multiply(RES, PSFCorrelated); var bMap = FFT.Backward(BMAPFourier, BMAPFourier.Length); //FFT.Shift(bMap); FitsIO.Write(bMap, "bMap.fits"); var PSF = FFT.Forward(CommonDeprecated.Residuals.Pad(psf, psf.GetLength(0), psf.GetLength(1)), 1.0); var PSF2Fourier = Common.Fourier2D.Multiply(PSF, PSFCorrelated); var xDiff = new double[xImage.GetLength(0), xImage.GetLength(1)]; //var blockInversion = CalcBlock(psf, yBlockSize, xBlockSize).Inverse(); var random = new Random(123); var lipschitz = ApproximateLipschitz(psf, yBlockSize, xBlockSize); var startL2 = NaiveGreedyCD.CalcDataObjective(residuals); var iter = 0; while (iter < maxIteration) { /* * var yB = random.Next(xImage.GetLength(0) / yBlockSize); * var xB = random.Next(xImage.GetLength(1) / xBlockSize); * yB = 64 / yBlockSize; * xB = 64 / xBlockSize; * var block = CopyFrom(bMap, yB, xB, yBlockSize, xBlockSize); * * //var optimized = block * blockInversion; * * var optimized = block / lipschitz /4; * var xOld = CopyFrom(xImage, yB, xB, yBlockSize, xBlockSize); * optimized = xOld + optimized; * * //shrink * for (int i = 0; i < optimized.Count; i++) * optimized[i] = Common.ShrinkElasticNet(optimized[i], lambda, alpha); * var optDiff = optimized - xOld; * //AddInto(xDiff, optDiff, yB, xB, yBlockSize, xBlockSize); * AddInto(xImage, optDiff, yB, xB, yBlockSize, xBlockSize); * FitsIO.Write(xImage, "xImageBlock.fits"); * FitsIO.Write(xDiff, "xDiff.fits"); * xDiff[64, 64] = 1.0; * * //update b-map * var XDIFF = FFT.Forward(xDiff); * XDIFF = Common.Fourier2D.Multiply(XDIFF, PSF2Fourier); * Common.Fourier2D.SubtractInPlace(BMAPFourier, XDIFF); * bMap = FFT.Backward(BMAPFourier, BMAPFourier.Length); * //FFT.Shift(bMap); * FitsIO.Write(bMap, "bMap2.fits"); * * //calc residuals for debug purposes * var XDIFF2 = FFT.Forward(xDiff); * XDIFF2 = Common.Fourier2D.Multiply(XDIFF2, PSF); * var RES2 = Common.Fourier2D.Subtract(RES, XDIFF2); * var resDebug = FFT.Backward(RES2, RES2.Length); * var L2 = NaiveGreedyCD.CalcDataObjective(resDebug); * FitsIO.Write(resDebug, "resDebug.fits"); * var xDiff3 = FFT.Backward(XDIFF2, XDIFF2.Length); * FitsIO.Write(xDiff3, "recDebug.fits"); * * //clear from xDiff * AddInto(xDiff, -optDiff, yB, xB, yBlockSize, xBlockSize);*/ iter++; } return(false); }
public static LocalDataset LoadTinyMeerKAT2(int rank, int nodeCount, string folder) { var blSum = 0; var blFileScans = new int[8]; var blFileCounts = new int[8]; for (int i = 0; i < 8; i++) { blFileCounts[i] = FitsIO.CountBaselines(Path.Combine(folder, "uvw" + i + ".fits")); blSum += blFileCounts[i]; blFileScans[i] = blSum; } var frequencies = FitsIO.ReadFrequencies(Path.Combine(folder, "freq.fits")); var blBeginIdx = rank * (int)(blSum / (double)nodeCount); var blEndIdx = rank + 1 < nodeCount ? (rank + 1) * (int)(blSum / (double)nodeCount) : blSum; var baselineCount = blEndIdx - blBeginIdx; LocalDataset output = null; for (int i = 0; i < 8; i++) { if (blBeginIdx < blFileScans[i]) { var blBefore = i > 0 ? blFileScans[i - 1] : 0; var start = blBeginIdx - blBefore; var end = Math.Min(start + baselineCount, blFileCounts[i]); var uvw = FitsIO.ReadUVW(Path.Combine(folder, "uvw" + i + ".fits"), start, end); var flags = FitsIO.ReadFlags(Path.Combine(folder, "flags" + i + ".fits"), start, end, uvw.GetLength(1), frequencies.Length); var visibilities = FitsIO.ReadVisibilities(Path.Combine(folder, "vis" + i + ".fits"), start, end, uvw.GetLength(1), frequencies.Length, 2.0); if (blBeginIdx + baselineCount > blFileCounts[i]) { //continue reading files until all baselines, which belong to the current node, are loaded var baselinesLoaded = end - start; for (int j = i + 1; j < 8; j++) { var end2 = Math.Min(baselineCount - baselinesLoaded, blFileCounts[j]); var uvw0 = FitsIO.ReadUVW(Path.Combine(folder, "uvw" + j + ".fits"), 0, end2); var flags0 = FitsIO.ReadFlags(Path.Combine(folder, "flags" + j + ".fits"), 0, end2, uvw.GetLength(1), frequencies.Length); var visibilities0 = FitsIO.ReadVisibilities(Path.Combine(folder, "vis" + j + ".fits"), 0, end2, uvw.GetLength(1), frequencies.Length, 2.0); uvw = FitsIO.Stitch(uvw, uvw0); flags = FitsIO.Stitch(flags, flags0); visibilities = FitsIO.Stitch(visibilities, visibilities0); baselinesLoaded += end2; if (baselinesLoaded >= baselineCount) { //last file read; break; } } } output = new LocalDataset(frequencies, uvw, flags, visibilities); break; } } return(output); }
private static void ReconstructMinorCycle(MeasurementData input, GriddingConstants c, int cutFactor, float[,] fullPsf, string folder, string file, int minorCycles, float searchPercent, bool useAccelerated = true, int blockSize = 1, int maxCycle = 6) { var metadata = Partitioner.CreatePartition(c, input.UVW, input.Frequencies); var totalSize = new Rectangle(0, 0, c.GridSize, c.GridSize); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var sidelobeHalf = PSF.CalcMaxSidelobe(fullPsf, 2); var random = new Random(123); var approx = new ApproxFast(totalSize, psfCut, 8, blockSize, 0.1f, searchPercent, false, useAccelerated); using (var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfCut, totalSize), new Rectangle(0, 0, psfCut.GetLength(0), psfCut.GetLength(1)))) using (var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1)))) using (var residualsConvolver = new PaddedConvolver(totalSize, fullPsf)) { var currentBMapCalculator = bMapCalculator; var maxLipschitz = PSF.CalcMaxLipschitz(psfCut); var lambda = (float)(LAMBDA * maxLipschitz); var lambdaTrue = (float)(LAMBDA * PSF.CalcMaxLipschitz(fullPsf)); var alpha = ALPHA; ApproxFast.LAMBDA_TEST = lambdaTrue; ApproxFast.ALPHA_TEST = alpha; var switchedToOtherPsf = false; var writer = new StreamWriter(folder + "/" + file + "_lambda.txt"); var data = new ApproxFast.TestingData(new StreamWriter(folder + "/" + file + ".txt")); var xImage = new float[c.GridSize, c.GridSize]; var residualVis = input.Visibilities; for (int cycle = 0; cycle < maxCycle; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(c, metadata, residualVis, input.UVW, input.Frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + "/dirty" + cycle + ".fits"); var minLambda = 0.0f; var dirtyCopy = Copy(dirtyImage); var xCopy = Copy(xImage); var currentLambda = 0f; //var residualsConvolver = new PaddedConvolver(PSF.CalcPaddedFourierConvolution(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))); for (int minorCycle = 0; minorCycle < minorCycles; minorCycle++) { FitsIO.Write(dirtyImage, folder + "/dirtyMinor_" + minorCycle + ".fits"); var maxDirty = Residuals.GetMax(dirtyImage); var bMap = currentBMapCalculator.Convolve(dirtyImage); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * maxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = (float)Math.Max(currentSideLobe / alpha, lambda); if (minorCycle == 0) { minLambda = (float)(maxB * sidelobeHalf * correctionFactor / alpha); } if (currentLambda < minLambda) { currentLambda = minLambda; } writer.WriteLine(cycle + ";" + minorCycle + ";" + currentLambda + ";" + minLambda); writer.Flush(); approx.DeconvolveTest(data, cycle, minorCycle, xImage, dirtyImage, psfCut, fullPsf, currentLambda, alpha, random, 15, 1e-5f); FitsIO.Write(xImage, folder + "/xImageMinor_" + minorCycle + ".fits"); if (currentLambda == lambda | currentLambda == minLambda) { break; } Console.WriteLine("resetting residuals!!"); //reset dirtyImage with full PSF var residualsUpdate = new float[xImage.GetLength(0), xImage.GetLength(1)]; Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { residualsUpdate[i, j] = xImage[i, j] - xCopy[i, j]; } }); residualsConvolver.ConvolveInPlace(residualsUpdate); Parallel.For(0, xCopy.GetLength(0), (i) => { for (int j = 0; j < xCopy.GetLength(1); j++) { dirtyImage[i, j] = dirtyCopy[i, j] - residualsUpdate[i, j]; } }); } if (currentLambda == lambda & !switchedToOtherPsf) { approx.ResetAMap(fullPsf); currentBMapCalculator = bMapCalculator2; lambda = lambdaTrue; switchedToOtherPsf = true; writer.WriteLine("switched"); writer.Flush(); } FitsIO.Write(xImage, folder + "/xImage_" + cycle + ".fits"); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(c, metadata, xGrid, input.UVW, input.Frequencies); residualVis = Visibilities.Substract(input.Visibilities, modelVis, input.Flags); } writer.Close(); } }
public static bool Deconvolve2(double[,] xImage, double[,] residuals, double[,] psf, double lambda, double alpha, Random random, int maxIteration = 100, double epsilon = 1e-4) { var xImage2 = ToFloatImage(xImage); var PSFConvolution = CommonDeprecated.PSF.CalcPaddedFourierConvolution(psf, residuals.GetLength(0), residuals.GetLength(1)); var PSFCorrelation = CommonDeprecated.PSF.CalculateFourierCorrelation(psf, residuals.GetLength(0), residuals.GetLength(1)); var PSFSquared = Fourier2D.Multiply(PSFConvolution, PSFCorrelation); var bMapCalculator = new PaddedConvolver(PSFCorrelation, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var resUpdateCalculator = new PaddedConvolver(PSFConvolution, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var bMapUpdateCalculator = new PaddedConvolver(PSFSquared, new Rectangle(0, 0, psf.GetLength(0), psf.GetLength(1))); var yBlockSize = 2; var xBlockSize = 2; var bMap = ToFloatImage(residuals); bMapCalculator.ConvolveInPlace(bMap); FitsIO.Write(bMap, "bmapFirst.fits"); var xDiff = new float[xImage.GetLength(0), xImage.GetLength(1)]; var lipschitz = ApproximateLipschitz(psf, yBlockSize, xBlockSize); var startL2 = NaiveGreedyCD.CalcDataObjective(residuals); var iter = 0; while (iter < maxIteration) { var yB = random.Next(xImage.GetLength(0) / yBlockSize); var xB = random.Next(xImage.GetLength(1) / xBlockSize); //yB = 64 / yBlockSize; //xB = 64 / xBlockSize; var block = CopyFrom(bMap, yB, xB, yBlockSize, xBlockSize); //var optimized = block * blockInversion; var update = block / lipschitz; var xOld = CopyFrom(xImage2, yB, xB, yBlockSize, xBlockSize); var optimized = xOld + update; //shrink bool containsNonZero = false; for (int i = 0; i < optimized.Count; i++) { optimized[i] = CommonDeprecated.ShrinkElasticNet(optimized[i], lambda, alpha); containsNonZero |= (optimized[i] - xOld[i]) != 0.0; } var optDiff = optimized - xOld; if (containsNonZero) { AddInto(xDiff, optDiff, yB, xB, yBlockSize, xBlockSize); AddInto(xImage2, optDiff, yB, xB, yBlockSize, xBlockSize); //FitsIO.Write(xImage2, "xImageBlock.fits"); //FitsIO.Write(xDiff, "xDiff.fits"); //update b-map bMapUpdateCalculator.ConvolveInPlace(xDiff); //FitsIO.Write(xDiff, "bMapUpdate.fits"); for (int i = 0; i < xDiff.GetLength(0); i++) { for (int j = 0; j < xDiff.GetLength(1); j++) { bMap[i, j] -= xDiff[i, j]; xDiff[i, j] = 0; } } //FitsIO.Write(bMap, "bMap2.fits"); //calc residuals for debug purposes AddInto(xDiff, optDiff, yB, xB, yBlockSize, xBlockSize); resUpdateCalculator.ConvolveInPlace(xDiff); //FitsIO.Write(xDiff, "residualsUpdate.fits"); for (int i = 0; i < xDiff.GetLength(0); i++) { for (int j = 0; j < xDiff.GetLength(1); j++) { residuals[i, j] -= xDiff[i, j]; xDiff[i, j] = 0; } } //FitsIO.Write(residuals, "residuals2.fits"); var l2 = NaiveGreedyCD.CalcDataObjective(residuals); Console.WriteLine(l2); } iter++; } for (int i = 0; i < xImage.GetLength(0); i++) { for (int j = 0; j < xImage.GetLength(1); j++) { xImage[i, j] = xImage2[i, j]; } } return(false); }
public static void Run() { var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = MeasurementData.LoadLMC(folder); int gridSize = 3072; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 1.5 / 3600.0 * PI / 180.0; int wLayerCount = 24; var maxW = 0.0; for (int i = 0; i < data.UVW.GetLength(0); i++) { for (int j = 0; j < data.UVW.GetLength(1); j++) { maxW = Math.Max(maxW, Math.Abs(data.UVW[i, j, 2])); } } maxW = Partitioner.MetersToLambda(maxW, data.Frequencies[data.Frequencies.Length - 1]); double wStep = maxW / (wLayerCount); var c = new GriddingConstants(data.VisibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep); var metadata = Partitioner.CreatePartition(c, data.UVW, data.Frequencies); var psfVis = new Complex[data.UVW.GetLength(0), data.UVW.GetLength(1), data.Frequencies.Length]; for (int i = 0; i < data.Visibilities.GetLength(0); i++) { for (int j = 0; j < data.Visibilities.GetLength(1); j++) { for (int k = 0; k < data.Visibilities.GetLength(2); k++) { if (!data.Flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(c, metadata, psfVis, data.UVW, data.Frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, c.VisibilitiesCount); FFT.Shift(psf); var outFolder = "ApproxExperiment"; Directory.CreateDirectory(outFolder); FitsIO.Write(psf, Path.Combine(outFolder, "psfFull.fits")); //tryout with simply cutting the PSF RunPsfSize(data, c, psf, outFolder); //RunBlocksize(data, c, psf, outFolder); RunSearchPercent(data, c, psf, outFolder); //RunNotAccelerated(data, c, psf, outFolder); }