public SimulationExample2_reconstructionfound Get() { var r = new V4D.Reconstruction.SimulationExample2_reconstructionfound(); var info = new ReconstructionInfo(); var allLines = System.IO.File.ReadAllLines( "eiffel tower.txt"); var all = allLines.Select(x => new V4D.Reconstruction.ContentInfo() { SourceId = x }); var reconstruction1 = allLines.PickRandom(240) .Select(x => new V4D.Reconstruction.ContentInfo() { SourceId = x }); //var reconstruction2 = allLines.PickRandom(12).Select(x => new V4D.Reconstruction.ContentInfo() {SourceId = x}); info.UsedContent.AddRange(reconstruction1); info.InputContent.AddRange(all); info.ReconstructionId = new UUID { Id = "<reconstruction id>" }; info.ReconstructionGroupId = new UUID { Id = "<reconstruction group id>" }; info.VisualAnalysisTags.Add("Tower"); r.Reconstructions.Add(info); //var t = Task.FromResult(r); return(r); }
public static void RunSpeedLarge() { var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = LMC.Load(folder); int gridSize = 3072; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 1.5 / 3600.0 * PI / 180.0; int wLayerCount = 24; var maxW = 0.0; for (int i = 0; i < data.uvw.GetLength(0); i++) { for (int j = 0; j < data.uvw.GetLength(1); j++) { maxW = Math.Max(maxW, Math.Abs(data.uvw[i, j, 2])); } } maxW = Partitioner.MetersToLambda(maxW, data.frequencies[data.frequencies.Length - 1]); var visCount2 = 0; for (int i = 0; i < data.flags.GetLength(0); i++) { for (int j = 0; j < data.flags.GetLength(1); j++) { for (int k = 0; k < data.flags.GetLength(2); k++) { if (!data.flags[i, j, k]) { visCount2++; } } } } double wStep = maxW / (wLayerCount); data.c = new GriddingConstants(data.visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep); data.metadata = Partitioner.CreatePartition(data.c, data.uvw, data.frequencies); var psfVis = new Complex[data.uvw.GetLength(0), data.uvw.GetLength(1), data.frequencies.Length]; for (int i = 0; i < data.visibilities.GetLength(0); i++) { for (int j = 0; j < data.visibilities.GetLength(1); j++) { for (int k = 0; k < data.visibilities.GetLength(2); k++) { if (!data.flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(data.c, data.metadata, psfVis, data.uvw, data.frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, data.c.VisibilitiesCount); FFT.Shift(psf); Directory.CreateDirectory("PSFSpeedExperimentApproxDeconv"); FitsIO.Write(psf, "psfFull.fits"); //tryout with simply cutting the PSF ReconstructionInfo experimentInfo = null; var psfCuts = new int[] { 2, 8, 16, 32, 64 }; var outFolder = "PSFSpeedExperimentApproxDeconv"; outFolder += @"\"; var fileHeader = "cycle;lambda;sidelobe;maxPixel;dataPenalty;regPenalty;currentRegPenalty;converged;iterCount;ElapsedTime"; /* * foreach (var cut in psfCuts) * { * using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) * { * writer.WriteLine(fileHeader); * experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 8, cut+"dirty", cut+"x", writer, 0.0, 1e-5f, false); * File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); * } * }*/ Directory.CreateDirectory("PSFSpeedExperimentApproxPSF"); outFolder = "PSFSpeedExperimentApproxPSF"; outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 8, cut + "dirty", cut + "x", writer, 0.0, 1e-5f, true); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } }
private static ReconstructionInfo ReconstructGradientApprox(Data input, float[,] fullPsf, string folder, int cutFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon) { var info = new ReconstructionInfo(); var psfCut = PSF.Cut(fullPsf, cutFactor); var maxSidelobe = PSF.CalcMaxSidelobe(fullPsf, cutFactor); var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var psfBMap = psfCut; var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(psfBMap, totalSize), new Rectangle(0, 0, psfBMap.GetLength(0), psfBMap.GetLength(1))); var bMapCalculator2 = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(fullPsf, totalSize), new Rectangle(0, 0, fullPsf.GetLength(0), fullPsf.GetLength(1))); var fastCD = new FastSerialCD(totalSize, psfCut); var fastCD2 = new FastSerialCD(totalSize, psfCut); fastCD2.ResetLipschitzMap(fullPsf); FitsIO.Write(psfCut, folder + cutFactor + "psf.fits"); var lambda = LAMBDA_GLOBAL * fastCD.MaxLipschitz; var lambdaTrue = (float)(LAMBDA_GLOBAL * PSF.CalcMaxLipschitz(fullPsf)); var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; DeconvolutionResult lastResult = null; var firstTimeConverged = false; var lastLambda = 0.0f; for (int cycle = 0; cycle < maxMajor; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits"); //calc data and reg penalty var dataPenalty = Residuals.CalcPenalty(dirtyImage); var regPenalty = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha); info.lastDataPenalty = dataPenalty; info.lastRegPenalty = regPenalty; var maxDirty = Residuals.GetMax(dirtyImage); var bMap = bMapCalculator.Convolve(dirtyImage); FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + ".fits"); var maxB = Residuals.GetMax(bMap); var correctionFactor = Math.Max(maxB / (maxDirty * fastCD.MaxLipschitz), 1.0f); var currentSideLobe = maxB * maxSidelobe * correctionFactor; var currentLambda = Math.Max(currentSideLobe / alpha, lambda); writer.Write(cycle + ";" + currentLambda + ";" + currentSideLobe + ";" + ";" + fastCD2.GetAbsMaxDiff(xImage, bMap, lambdaTrue, alpha) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";");; writer.Flush(); //check wether we can minimize the objective further with the current psf var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff; var minimumReached = (lastResult != null && lastResult.Converged && fastCD2.GetAbsMaxDiff(xImage, dirtyImage, lambdaTrue, alpha) < MAJOR_EPSILON && currentLambda == lambda); if (lambda == lastLambda & !firstTimeConverged) { firstTimeConverged = true; minimumReached = false; } if (!objectiveReached & !minimumReached) { //writer.Write(firstTimeConverged + ";"); //writer.Flush(); info.totalDeconv.Start(); if (!firstTimeConverged) { lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon); } else { bMap = bMapCalculator2.Convolve(dirtyImage); //FitsIO.Write(bMap, folder + dirtyPrefix + "bmap_" + cycle + "_full.fits"); maxB = Residuals.GetMax(bMap); correctionFactor = Math.Max(maxB / (maxDirty * fastCD2.MaxLipschitz), 1.0f); currentSideLobe = maxB * maxSidelobe * correctionFactor; currentLambda = Math.Max(currentSideLobe / alpha, lambdaTrue); info.totalDeconv.Start(); lastResult = fastCD.Deconvolve(xImage, bMap, currentLambda, alpha, 30000, epsilon); info.totalDeconv.Stop(); } info.totalDeconv.Stop(); FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits"); writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n"); writer.Flush(); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } else { writer.Write(false + ";0;0\n"); writer.Flush(); break; } lastLambda = currentLambda; } bMapCalculator.Dispose(); bMapCalculator2.Dispose(); return(info); }
public static void RunApproximationMethods() { var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = LMC.Load(folder); var rootFolder = Directory.GetCurrentDirectory(); var maxW = 0.0; for (int i = 0; i < data.uvw.GetLength(0); i++) { for (int j = 0; j < data.uvw.GetLength(1); j++) { maxW = Math.Max(maxW, Math.Abs(data.uvw[i, j, 2])); } } maxW = Partitioner.MetersToLambda(maxW, data.frequencies[data.frequencies.Length - 1]); var visCount2 = 0; for (int i = 0; i < data.flags.GetLength(0); i++) { for (int j = 0; j < data.flags.GetLength(1); j++) { for (int k = 0; k < data.flags.GetLength(2); k++) { if (!data.flags[i, j, k]) { visCount2++; } } } } var visibilitiesCount = visCount2; int gridSize = 3072; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 1.5 / 3600.0 * PI / 180.0; int wLayerCount = 24; double wStep = maxW / (wLayerCount); data.c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep); data.metadata = Partitioner.CreatePartition(data.c, data.uvw, data.frequencies); data.visibilitiesCount = visibilitiesCount; var psfVis = new Complex[data.uvw.GetLength(0), data.uvw.GetLength(1), data.frequencies.Length]; for (int i = 0; i < data.visibilities.GetLength(0); i++) { for (int j = 0; j < data.visibilities.GetLength(1); j++) { for (int k = 0; k < data.visibilities.GetLength(2); k++) { if (!data.flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(data.c, data.metadata, psfVis, data.uvw, data.frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, data.c.VisibilitiesCount); FFT.Shift(psf); Directory.CreateDirectory("PSFSizeExperimentLarge"); Directory.SetCurrentDirectory("PSFSizeExperimentLarge"); FitsIO.Write(psf, "psfFull.fits"); Console.WriteLine(PSF.CalcMaxLipschitz(psf)); Console.WriteLine(visCount2); //reconstruct with full psf and find reference objective value var fileHeader = "cycle;lambda;sidelobe;maxPixel;dataPenalty;regPenalty;currentRegPenalty;converged;iterCount;ElapsedTime"; var objectiveCutoff = REFERENCE_L2_PENALTY + REFERENCE_ELASTIC_PENALTY; var recalculateFullPSF = true; if (recalculateFullPSF) { ReconstructionInfo referenceInfo = null; using (var writer = new StreamWriter("1Psf.txt", false)) { writer.WriteLine(fileHeader); referenceInfo = ReconstructSimple(data, psf, "", 1, 12, "dirtyReference", "xReference", writer, 0.0, 1e-5f, false); File.WriteAllText("1PsfTotal.txt", referenceInfo.totalDeconv.Elapsed.ToString()); } objectiveCutoff = referenceInfo.lastDataPenalty + referenceInfo.lastRegPenalty; } //tryout with simply cutting the PSF ReconstructionInfo experimentInfo = null; var psfCuts = new int[] { 16, 32 }; var outFolder = "cutPsf"; Directory.CreateDirectory(outFolder); outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 12, cut + "dirty", cut + "x", writer, 0.0, 1e-5f, false); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } //Tryout with cutting the PSF, but starting from the true bMap outFolder = "cutPsf2"; Directory.CreateDirectory(outFolder); outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructSimple(data, psf, outFolder, cut, 12, cut + "dirty", cut + "x", writer, 0.0, 1e-5f, true); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } //combined, final solution. Cut the psf in half, optimize until convergence, and then do one more major cycle with the second method outFolder = "properSolution"; Directory.CreateDirectory(outFolder); outFolder += @"\"; foreach (var cut in psfCuts) { using (var writer = new StreamWriter(outFolder + cut + "Psf.txt", false)) { writer.WriteLine(fileHeader); experimentInfo = ReconstructGradientApprox(data, psf, outFolder, cut, 12, cut + "dirty", cut + "x", writer, 0.0, 1e-5f); File.WriteAllText(outFolder + cut + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } Directory.SetCurrentDirectory(rootFolder); }
public static void Run() { var folder = @"C:\dev\GitHub\p9-data\large\fits\meerkat_tiny\"; var data = LMC.Load(folder); var rootFolder = Directory.GetCurrentDirectory(); var maxW = 0.0; for (int i = 0; i < data.uvw.GetLength(0); i++) { for (int j = 0; j < data.uvw.GetLength(1); j++) { maxW = Math.Max(maxW, Math.Abs(data.uvw[i, j, 2])); } } maxW = Partitioner.MetersToLambda(maxW, data.frequencies[data.frequencies.Length - 1]); var visCount2 = 0; for (int i = 0; i < data.flags.GetLength(0); i++) { for (int j = 0; j < data.flags.GetLength(1); j++) { for (int k = 0; k < data.flags.GetLength(2); k++) { if (!data.flags[i, j, k]) { visCount2++; } } } } var visibilitiesCount = visCount2; int gridSize = 2048; int subgridsize = 32; int kernelSize = 16; int max_nr_timesteps = 1024; double cellSize = 2.0 / 3600.0 * PI / 180.0; int wLayerCount = 32; double wStep = maxW / (wLayerCount); data.c = new GriddingConstants(visibilitiesCount, gridSize, subgridsize, kernelSize, max_nr_timesteps, (float)cellSize, wLayerCount, wStep); data.metadata = Partitioner.CreatePartition(data.c, data.uvw, data.frequencies); data.visibilitiesCount = visibilitiesCount; var psfVis = new Complex[data.uvw.GetLength(0), data.uvw.GetLength(1), data.frequencies.Length]; for (int i = 0; i < data.visibilities.GetLength(0); i++) { for (int j = 0; j < data.visibilities.GetLength(1); j++) { for (int k = 0; k < data.visibilities.GetLength(2); k++) { if (!data.flags[i, j, k]) { psfVis[i, j, k] = new Complex(1.0, 0); } else { psfVis[i, j, k] = new Complex(0, 0); } } } } Console.WriteLine("gridding psf"); var psfGrid = IDG.GridW(data.c, data.metadata, psfVis, data.uvw, data.frequencies); var psf = FFT.WStackIFFTFloat(psfGrid, data.c.VisibilitiesCount); FFT.Shift(psf); var objectiveCutoff = REFERENCE_L2_PENALTY + REFERENCE_ELASTIC_PENALTY; var actualLipschitz = (float)PSF.CalcMaxLipschitz(psf); Console.WriteLine("Calc Histogram"); var histPsf = GetHistogram(psf, 256, 0.05f); var experiments = new float[] { 0.5f, /*0.4f, 0.2f, 0.1f, 0.05f*/ }; Console.WriteLine("Done Histogram"); Directory.CreateDirectory("PSFMask"); Directory.SetCurrentDirectory("PSFMask"); FitsIO.Write(psf, "psfFull.fits"); //reconstruct with full psf and find reference objective value ReconstructionInfo experimentInfo = null; var outFolder = ""; var fileHeader = "cycle;lambda;sidelobe;dataPenalty;regPenalty;currentRegPenalty;converged;iterCount;ElapsedTime"; foreach (var maskPercent in experiments) { using (var writer = new StreamWriter(outFolder + maskPercent + "Psf.txt", false)) { var maskedPSF = Common.Copy(psf); var maskedPixels = MaskPSF(maskedPSF, histPsf, maskPercent); writer.WriteLine(maskedPixels + ";" + maskedPixels / (double)maskedPSF.Length); FitsIO.Write(maskedPSF, outFolder + maskPercent + "Psf.fits"); writer.WriteLine(fileHeader); writer.Flush(); experimentInfo = Reconstruct(data, actualLipschitz, maskedPSF, outFolder, 1, 10, maskPercent + "dirty", maskPercent + "x", writer, objectiveCutoff, 1e-5f, false); File.WriteAllText(outFolder + maskPercent + "PsfTotal.txt", experimentInfo.totalDeconv.Elapsed.ToString()); } } Directory.SetCurrentDirectory(rootFolder); }
private static ReconstructionInfo Reconstruct(Data input, float fullLipschitz, float[,] maskedPsf, string folder, float maskFactor, int maxMajor, string dirtyPrefix, string xImagePrefix, StreamWriter writer, double objectiveCutoff, float epsilon, bool maskPsf2) { var info = new ReconstructionInfo(); var totalSize = new Rectangle(0, 0, input.c.GridSize, input.c.GridSize); var bMapCalculator = new PaddedConvolver(PSF.CalcPaddedFourierCorrelation(maskedPsf, totalSize), new Rectangle(0, 0, maskedPsf.GetLength(0), maskedPsf.GetLength(1))); var maskedPsf2 = PSF.CalcPSFSquared(maskedPsf); if (maskPsf2) { Mask(maskedPsf2, 1e-5f); } writer.WriteLine((CountNonZero(maskedPsf2) - maskedPsf2.Length) / (double)maskedPsf2.Length); var fastCD = new FastSerialCD(totalSize, totalSize, maskedPsf, maskedPsf2); FitsIO.Write(maskedPsf, folder + maskFactor + "psf.fits"); var lambda = 0.4f * fastCD.MaxLipschitz; var lambdaTrue = 0.4f * fullLipschitz; var alpha = 0.1f; var xImage = new float[input.c.GridSize, input.c.GridSize]; var residualVis = input.visibilities; DeconvolutionResult lastResult = null; for (int cycle = 0; cycle < maxMajor; cycle++) { Console.WriteLine("cycle " + cycle); var dirtyGrid = IDG.GridW(input.c, input.metadata, residualVis, input.uvw, input.frequencies); var dirtyImage = FFT.WStackIFFTFloat(dirtyGrid, input.c.VisibilitiesCount); FFT.Shift(dirtyImage); FitsIO.Write(dirtyImage, folder + dirtyPrefix + cycle + ".fits"); //calc data and reg penalty var dataPenalty = Residuals.CalcPenalty(dirtyImage); var regPenalty = ElasticNet.CalcPenalty(xImage, lambdaTrue, alpha); var regPenaltyCurrent = ElasticNet.CalcPenalty(xImage, lambda, alpha); info.lastDataPenalty = dataPenalty; info.lastRegPenalty = regPenalty; bMapCalculator.ConvolveInPlace(dirtyImage); //FitsIO.Write(dirtyImage, folder + dirtyPrefix + "bmap_" + cycle + ".fits"); var currentLambda = lambda; writer.Write(cycle + ";" + currentLambda + ";" + Residuals.GetMax(dirtyImage) + ";" + dataPenalty + ";" + regPenalty + ";" + regPenaltyCurrent + ";"); writer.Flush(); //check wether we can minimize the objective further with the current psf var objectiveReached = (dataPenalty + regPenalty) < objectiveCutoff; var minimumReached = (lastResult != null && lastResult.IterationCount < 100 && lastResult.Converged); if (!objectiveReached & !minimumReached) { info.totalDeconv.Start(); lastResult = fastCD.Deconvolve(xImage, dirtyImage, currentLambda, alpha, 50000, epsilon); info.totalDeconv.Stop(); FitsIO.Write(xImage, folder + xImagePrefix + cycle + ".fits"); writer.Write(lastResult.Converged + ";" + lastResult.IterationCount + ";" + lastResult.ElapsedTime.TotalSeconds + "\n"); writer.Flush(); FFT.Shift(xImage); var xGrid = FFT.Forward(xImage); FFT.Shift(xImage); var modelVis = IDG.DeGridW(input.c, input.metadata, xGrid, input.uvw, input.frequencies); residualVis = Visibilities.Substract(input.visibilities, modelVis, input.flags); } else { writer.Write(false + ";0;0"); writer.Flush(); break; } } return(info); }