public void lbfgsTest()
{
Func<double[], double> f = rosenbrockFunction;
Func<double[], double[]> g = rosenbrockGradient;
Assert.AreEqual(104, f(new[] { -1.0, 2.0 }));
int n = 2; // number of variables
double[] initial = { -1.2, 1 };
BroydenFletcherGoldfarbShanno lbfgs = new BroydenFletcherGoldfarbShanno(n, f, g);
double actual = lbfgs.Minimize(initial);
double expected = 0;
Assert.AreEqual(expected, actual, 1e-10);
double[] result = lbfgs.Solution;
Assert.AreEqual(49, lbfgs.Evaluations);
Assert.AreEqual(40, lbfgs.Iterations);
Assert.AreEqual(0.99999999999963229, result[0]);
Assert.AreEqual(0.99999999999924027, result[1]);
double y = f(result);
double[] d = g(result);
Assert.AreEqual(1.9432410039142452E-25, y);
Assert.AreEqual(0.0000000000089901419642010907, d[0]);
Assert.AreEqual(-0.0000000000048627768478581856, d[1]);
}
public void ConstructorTest2()
{
Function function = // min f(x) = 10 * (x+1)^2 + y^2
x => 10.0 * Math.Pow(x[0] + 1.0, 2.0) + Math.Pow(x[1], 2.0);
Gradient gradient = x => new[] { 20 * (x[0] + 1), 2 * x[1] };
double[] start = new double[2];
BroydenFletcherGoldfarbShanno target = new BroydenFletcherGoldfarbShanno(2,
function.Invoke, gradient.Invoke);
Assert.IsTrue(target.Minimize());
double minimum = target.Value;
double[] solution = target.Solution;
Assert.AreEqual(0, minimum, 1e-10);
Assert.AreEqual(-1, solution[0], 1e-5);
Assert.AreEqual(0, solution[1], 1e-5);
double expectedMinimum = function(target.Solution);
Assert.AreEqual(expectedMinimum, minimum);
}
public void lbfgsTest()
{
Func<double[], double> f = rosenbrockFunction;
Func<double[], double[]> g = rosenbrockGradient;
Assert.AreEqual(104, f(new[] { -1.0, 2.0 }));
int n = 2; // number of variables
double[] initial = { -1.2, 1 };
var lbfgs = new BroydenFletcherGoldfarbShanno(n, f, g);
double expected = 0;
Assert.IsTrue(lbfgs.Minimize(initial));
bool success = lbfgs.Minimize();
double actual = lbfgs.Value;
Assert.IsTrue(success);
Assert.AreEqual(expected, actual, 1e-10);
double[] result = lbfgs.Solution;
Assert.AreEqual(1.0, result[0], 1e-5);
Assert.AreEqual(1.0, result[1], 1e-5);
double y = f(result);
double[] d = g(result);
Assert.AreEqual(0, y, 1e-10);
Assert.AreEqual(0, d[0], 1e-6);
Assert.AreEqual(0, d[1], 1e-6);
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="numberOfVariables">The number of free parameters in the optimization problem.</param>
/// <param name="constraints">The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangianSolver(int numberOfVariables,
IEnumerable <NonlinearConstraint> constraints)
{
var innerSolver = new BroydenFletcherGoldfarbShanno(numberOfVariables);
init(numberOfVariables, constraints, innerSolver);
}
private void init(NonlinearObjectiveFunction function,
IEnumerable <NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
{
if (function != null)
{
if (function.NumberOfVariables != NumberOfVariables)
{
throw new ArgumentOutOfRangeException("function",
"Incorrect number of variables in the objective function. " +
"The number of variables must match the number of variables set in the solver.");
}
this.Function = function.Function;
this.Gradient = function.Gradient;
}
if (innerSolver == null)
{
innerSolver = new BroydenFletcherGoldfarbShanno(NumberOfVariables)
{
LineSearch = Optimization.LineSearch.BacktrackingArmijo,
Corrections = 10
};
}
List <NonlinearConstraint> equality = new List <NonlinearConstraint>();
List <NonlinearConstraint> lesserThan = new List <NonlinearConstraint>();
List <NonlinearConstraint> greaterThan = new List <NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.lambda = new double[equalityConstraints.Length];
this.mu = new double[lesserThanConstraints.Length];
this.nu = new double[greaterThanConstraints.Length];
this.dualSolver = innerSolver;
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
}
public static void Test()
{
var m = Matrix.Create(1, 10, 1.0);
var v = Matrix.Create(10, 1, 1.0);
var p = m.Multiply(v);
Func<double[], double> f = (x) =>
-Math.Exp(-Math.Pow(x[0] - 1, 2)) - Math.Exp(-0.5*Math.Pow(x[1] - 2, 2));
Func<double[], double[]> g = (x) => new[]
{
// df/dx = {-2 e^(- (x-1)^2) (x-1)}
2*Math.Exp(-Math.Pow(x[0] - 1, 2))*(x[0] - 1),
// df/dy = {- e^(-1/2 (y-2)^2) (y-2)}
Math.Exp(-0.5*Math.Pow(x[1] - 2, 2))*(x[1] - 2)
};
var lbfgs = new BroydenFletcherGoldfarbShanno(2, f, g);
lbfgs.Progress += (sender, args) =>
{
//var grad = args.Gradient;
//var sol = args.Solution;
//var step = args.Step;
Console.WriteLine("Iteration: {0}, solution: {1}, {2}", args.Iteration, args.Solution[0], args.Solution[1]);
};
bool runAgain;
do
{
try
{
lbfgs.Minimize();
runAgain = false;
}
catch (LineSearchFailedException)
{
Console.WriteLine("Failed");
runAgain = true;
}
} while (runAgain);
double minValue = lbfgs.Value;
double[] solution = lbfgs.Solution;
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="numberOfVariables">The number of free parameters in the optimization problem.</param>
/// <param name="constraints">The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangianSolver(int numberOfVariables, IEnumerable <NonlinearConstraint> constraints)
{
this.numberOfVariables = numberOfVariables;
List <NonlinearConstraint> equality = new List <NonlinearConstraint>();
List <NonlinearConstraint> lesserThan = new List <NonlinearConstraint>();
List <NonlinearConstraint> greaterThan = new List <NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.Solution = new double[numberOfVariables];
dualSolver = new BroydenFletcherGoldfarbShanno(numberOfVariables);
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
//for (int i = 0; i < Solution.Length; i++)
// Solution[i] = Accord.Math.Tools.Random.NextDouble() * 2 - 1;
}
public virtual void ProcessCTF(MapHeader originalHeader, Image originalStack, bool doastigmatism, decimal scaleFactor)
{
if (!Directory.Exists(PowerSpectrumDir))
Directory.CreateDirectory(PowerSpectrumDir);
//CTF = new CTF();
PS1D = null;
_SimulatedBackground = null;
_SimulatedScale = new Cubic1D(new[] { new float2(0, 1), new float2(1, 1) });
#region Dimensions and grids
int NFrames = originalHeader.Dimensions.Z;
int2 DimsImage = new int2(originalHeader.Dimensions);
int2 DimsRegion = new int2(MainWindow.Options.CTFWindow, MainWindow.Options.CTFWindow);
float OverlapFraction = 0.5f;
int2 DimsPositionGrid;
int3[] PositionGrid = Helper.GetEqualGridSpacing(DimsImage, new int2(DimsRegion.X / 1, DimsRegion.Y / 1), OverlapFraction, out DimsPositionGrid);
int NPositions = (int)DimsPositionGrid.Elements();
int CTFGridX = Math.Min(DimsPositionGrid.X, MainWindow.Options.GridCTFX);
int CTFGridY = Math.Min(DimsPositionGrid.Y, MainWindow.Options.GridCTFY);
int CTFGridZ = Math.Min(NFrames, MainWindow.Options.GridCTFZ);
GridCTF = new CubicGrid(new int3(CTFGridX, CTFGridY, CTFGridZ));
GridCTFPhase = new CubicGrid(new int3(1, 1, CTFGridZ));
bool CTFSpace = CTFGridX * CTFGridY > 1;
bool CTFTime = CTFGridZ > 1;
int3 CTFSpectraGrid = new int3(CTFSpace ? DimsPositionGrid.X : 1,
CTFSpace ? DimsPositionGrid.Y : 1,
CTFTime ? CTFGridZ : 1);
int MinFreqInclusive = (int)(MainWindow.Options.CTFRangeMin * DimsRegion.X / 2);
int MaxFreqExclusive = (int)(MainWindow.Options.CTFRangeMax * DimsRegion.X / 2);
int NFreq = MaxFreqExclusive - MinFreqInclusive;
float PixelSize = (float)(MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5f;
float PixelDelta = (float)(MainWindow.Options.CTFPixelMax - MainWindow.Options.CTFPixelMin) * 0.5f;
float PixelAngle = (float)MainWindow.Options.CTFPixelAngle / 180f * (float)Math.PI;
#endregion
#region Allocate GPU memory
Image CTFSpectra = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.X, (int)CTFSpectraGrid.Elements()), true);
Image CTFMean = new Image(IntPtr.Zero, new int3(DimsRegion), true);
Image CTFCoordsCart = new Image(new int3(DimsRegion), true, true);
Image CTFCoordsPolarTrimmed = new Image(new int3(NFreq, DimsRegion.X, 1), false, true);
#endregion
// Extract movie regions, create individual spectra in Cartesian coordinates and their mean.
#region Create spectra
GPU.CreateSpectra(originalStack.GetDevice(Intent.Read),
DimsImage,
NFrames,
PositionGrid,
NPositions,
DimsRegion,
CTFSpectraGrid,
CTFSpectra.GetDevice(Intent.Write),
CTFMean.GetDevice(Intent.Write));
originalStack.FreeDevice(); // Won't need it in this method anymore.
#endregion
// Populate address arrays for later.
#region Init addresses
{
float2[] CoordsData = new float2[CTFCoordsCart.ElementsSliceComplex];
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) => CoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r, a));
CTFCoordsCart.UpdateHostWithComplex(new[] { CoordsData });
CoordsData = new float2[NFreq * DimsRegion.X];
Helper.ForEachElement(CTFCoordsPolarTrimmed.DimsSlice, (x, y) =>
{
float Angle = ((float)y / DimsRegion.X + 0.5f) * (float)Math.PI;
float Ny = 1f / DimsRegion.X;
CoordsData[y * NFreq + x] = new float2((x + MinFreqInclusive) * Ny, Angle);
});
CTFCoordsPolarTrimmed.UpdateHostWithComplex(new[] { CoordsData });
}
#endregion
// Retrieve average 1D spectrum from CTFMean (not corrected for astigmatism yet).
#region Initial 1D spectrum
{
Image CTFAverage1D = new Image(IntPtr.Zero, new int3(DimsRegion.X / 2, 1, 1));
GPU.CTFMakeAverage(CTFMean.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
(uint)CTFMean.ElementsSliceReal,
(uint)DimsRegion.X,
new[] { new CTF().ToStruct() },
new CTF().ToStruct(),
0,
(uint)DimsRegion.X / 2,
null,
1,
CTFAverage1D.GetDevice(Intent.Write));
//CTFAverage1D.WriteMRC("CTFAverage1D.mrc");
float[] CTFAverage1DData = CTFAverage1D.GetHost(Intent.Read)[0];
float2[] ForPS1D = new float2[DimsRegion.X / 2];
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, (float)Math.Round(CTFAverage1DData[i], 4));
_PS1D = ForPS1D;
CTFAverage1D.Dispose();
}
#endregion
#region Background fitting methods
Action UpdateBackgroundFit = () =>
{
float2[] ForPS1D = PS1D.Skip(Math.Max(5, MinFreqInclusive / 2)).ToArray();
Cubic1D.FitCTF(ForPS1D,
v => v.Select(x => CTF.Get1D(x / (float)CTF.PixelSize, true)).ToArray(),
CTF.GetZeros(),
CTF.GetPeaks(),
out _SimulatedBackground,
out _SimulatedScale);
};
Action<bool> UpdateRotationalAverage = keepbackground =>
{
float[] MeanData = CTFMean.GetHost(Intent.Read)[0];
Image CTFMeanCorrected = new Image(new int3(DimsRegion), true);
float[] MeanCorrectedData = CTFMeanCorrected.GetHost(Intent.Write)[0];
// Subtract current background estimate from spectra, populate coords.
Helper.ForEachElementFT(DimsRegion,
(x, y, xx, yy, r, a) =>
{
int i = y * (DimsRegion.X / 2 + 1) + x;
MeanCorrectedData[i] = MeanData[i] - _SimulatedBackground.Interp(r / DimsRegion.X);
});
Image CTFAverage1D = new Image(IntPtr.Zero, new int3(DimsRegion.X / 2, 1, 1));
GPU.CTFMakeAverage(CTFMeanCorrected.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
(uint)CTFMeanCorrected.DimsEffective.ElementsSlice(),
(uint)DimsRegion.X,
new[] { CTF.ToStruct() },
CTF.ToStruct(),
0,
(uint)DimsRegion.X / 2,
null,
1,
CTFAverage1D.GetDevice(Intent.Write));
//CTFAverage1D.WriteMRC("CTFAverage1D.mrc");
float[] RotationalAverageData = CTFAverage1D.GetHost(Intent.Read)[0];
float2[] ForPS1D = new float2[PS1D.Length];
if (keepbackground)
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, RotationalAverageData[i] + _SimulatedBackground.Interp((float)i / DimsRegion.X));
else
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, RotationalAverageData[i]);
MathHelper.UnNaN(ForPS1D);
_PS1D = ForPS1D;
CTFMeanCorrected.Dispose();
CTFAverage1D.Dispose();
};
#endregion
// Fit background to currently best average (not corrected for astigmatism yet).
{
float2[] ForPS1D = PS1D.Skip(MinFreqInclusive).Take(Math.Max(2, NFreq / 2)).ToArray();
int NumNodes = Math.Max(3, (int)((MainWindow.Options.CTFRangeMax - MainWindow.Options.CTFRangeMin) * 5M));
_SimulatedBackground = Cubic1D.Fit(ForPS1D, NumNodes); // This won't fit falloff and scale, because approx function is 0
float[] CurrentBackground = _SimulatedBackground.Interp(PS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq / 2).ToArray();
float[] Subtracted1D = new float[ForPS1D.Length];
for (int i = 0; i < ForPS1D.Length; i++)
Subtracted1D[i] = ForPS1D[i].Y - CurrentBackground[i];
MathHelper.NormalizeInPlace(Subtracted1D);
float ZMin = (float)MainWindow.Options.CTFZMin;
float ZMax = (float)MainWindow.Options.CTFZMax;
float ZStep = (ZMax - ZMin) / 100f;
float BestZ = 0, BestPhase = 0, BestScore = -999;
for (float z = ZMin; z <= ZMax + 1e-5f; z += ZStep)
{
for (float p = 0; p <= (MainWindow.Options.CTFDoPhase ? 1f : 0f); p += 0.01f)
{
CTF CurrentParams = new CTF
{
PixelSize = (MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5M,
Defocus = (decimal)z,
PhaseShift = (decimal)p,
Cs = MainWindow.Options.CTFCs,
Voltage = MainWindow.Options.CTFVoltage,
Amplitude = MainWindow.Options.CTFAmplitude
};
float[] SimulatedCTF = CurrentParams.Get1D(PS1D.Length, true).Skip(MinFreqInclusive).Take(Math.Max(2, NFreq / 2)).ToArray();
MathHelper.NormalizeInPlace(SimulatedCTF);
float Score = MathHelper.CrossCorrelate(Subtracted1D, SimulatedCTF);
if (Score > BestScore)
{
BestScore = Score;
BestZ = z;
BestPhase = p;
}
}
}
CTF = new CTF
{
PixelSize = (MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5M,
Defocus = (decimal)BestZ,
PhaseShift = (decimal)BestPhase,
Cs = MainWindow.Options.CTFCs,
Voltage = MainWindow.Options.CTFVoltage,
Amplitude = MainWindow.Options.CTFAmplitude
};
UpdateRotationalAverage(true); // This doesn't have a nice background yet.
UpdateBackgroundFit(); // Now get a reasonably nice background.
}
// Fit defocus, (phase shift), (astigmatism) to average background-subtracted spectrum,
// which is in polar coords at this point (for equal weighting of all frequencies).
#region Grid search
{
Image CTFMeanPolarTrimmed = CTFMean.AsPolar((uint)MinFreqInclusive, (uint)(MinFreqInclusive + NFreq / 1));
// Subtract current background.
Image CurrentBackground = new Image(_SimulatedBackground.Interp(PS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq / 1).ToArray());
CTFMeanPolarTrimmed.SubtractFromLines(CurrentBackground);
CurrentBackground.Dispose();
/*Image WaterMask = new Image(new int3(NFreq, 1, 1));
float[] WaterData = WaterMask.GetHost(Intent.Write)[0];
for (int i = 0; i < NFreq; i++)
{
float f = (i + MinFreqInclusive) / (float)DimsRegion.X * 2f;
WaterData[i] = f > 0.2f && f < 0.6f ? 0f : 1f;
}
//CTFMeanPolarTrimmed.MultiplyLines(WaterMask);
WaterMask.Dispose();*/
// Normalize for CC (not strictly needed, but it's converted for fp16 later, so let's be on the safe side of the fp16 range.
GPU.Normalize(CTFMeanPolarTrimmed.GetDevice(Intent.Read), CTFMeanPolarTrimmed.GetDevice(Intent.Write), (uint)CTFMeanPolarTrimmed.ElementsReal, 1);
//CTFMeanPolarTrimmed.WriteMRC("ctfmeanpolartrimmed.mrc");
CTF StartParams = new CTF
{
PixelSize = (MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5M,
PixelSizeDelta = Math.Abs(MainWindow.Options.CTFPixelMax - MainWindow.Options.CTFPixelMin),
PixelSizeAngle = MainWindow.Options.CTFPixelAngle,
Defocus = CTF.Defocus,// (MainWindow.Options.CTFZMin + MainWindow.Options.CTFZMax) * 0.5M,
DefocusDelta = doastigmatism ? 0 : MainWindow.Options.CTFAstigmatism,
DefocusAngle = doastigmatism ? 0 : MainWindow.Options.CTFAstigmatismAngle,
Cs = MainWindow.Options.CTFCs,
Voltage = MainWindow.Options.CTFVoltage,
Amplitude = MainWindow.Options.CTFAmplitude
};
CTFFitStruct FitParams = new CTFFitStruct
{
//Pixelsize = new float3(-0.02e-10f, 0.02e-10f, 0.01e-10f),
//Pixeldelta = new float3(0.0f, 0.02e-10f, 0.01e-10f),
//Pixelangle = new float3(0, 2 * (float)Math.PI, 1 * (float)Math.PI / 18),
//Defocus = new float3((float)(MainWindow.Options.CTFZMin - StartParams.Defocus) * 1e-6f,
// (float)(MainWindow.Options.CTFZMax - StartParams.Defocus) * 1e-6f,
// 0.025e-6f),
Defocus = new float3(-0.4e-6f,
0.4e-6f,
0.025e-6f),
Defocusdelta = doastigmatism ? new float3(0, 0.8e-6f, 0.02e-6f) : new float3(0, 0, 0),
Astigmatismangle = doastigmatism ? new float3(0, 2 * (float)Math.PI, 1 * (float)Math.PI / 18) : new float3(0, 0, 0),
Phaseshift = MainWindow.Options.CTFDoPhase ? new float3(0, (float)Math.PI, 0.025f * (float)Math.PI) : new float3(0, 0, 0)
};
CTFStruct ResultStruct = GPU.CTFFitMean(CTFMeanPolarTrimmed.GetDevice(Intent.Read),
CTFCoordsPolarTrimmed.GetDevice(Intent.Read),
CTFMeanPolarTrimmed.DimsSlice,
StartParams.ToStruct(),
FitParams,
doastigmatism);
CTF.FromStruct(ResultStruct);
CTF.Defocus = Math.Max(CTF.Defocus, MainWindow.Options.CTFZMin);
CTFMeanPolarTrimmed.Dispose();
UpdateRotationalAverage(true); // This doesn't have a nice background yet.
UpdateBackgroundFit(); // Now get a reasonably nice background.
UpdateRotationalAverage(true); // This time, with the nice background.
UpdateBackgroundFit(); // Make the background even nicer!
}
#endregion
/*for (int i = 0; i < PS1D.Length; i++)
PS1D[i].Y -= SimulatedBackground.Interp(PS1D[i].X);
SimulatedBackground = new Cubic1D(SimulatedBackground.Data.Select(v => new float2(v.X, 0f)).ToArray());
OnPropertyChanged("PS1D");
CTFSpectra.Dispose();
CTFMean.Dispose();
CTFCoordsCart.Dispose();
CTFCoordsPolarTrimmed.Dispose();
Simulated1D = GetSimulated1D();
CTFQuality = GetCTFQuality();
return;*/
// Do BFGS optimization of defocus, astigmatism and phase shift,
// using 2D simulation for comparison
#region BFGS
bool[] CTFSpectraConsider = new bool[CTFSpectraGrid.Elements()];
for (int i = 0; i < CTFSpectraConsider.Length; i++)
CTFSpectraConsider[i] = true;
int NCTFSpectraConsider = CTFSpectraConsider.Length;
GridCTF = new CubicGrid(GridCTF.Dimensions, (float)CTF.Defocus, (float)CTF.Defocus, Dimension.X);
GridCTFPhase = new CubicGrid(GridCTFPhase.Dimensions, (float)CTF.PhaseShift, (float)CTF.PhaseShift, Dimension.X);
for (int preciseFit = 2; preciseFit < 3; preciseFit++)
{
NFreq = (MaxFreqExclusive - MinFreqInclusive) * (preciseFit + 1) / 3;
//if (preciseFit >= 2)
// NFreq = MaxFreqExclusive - MinFreqInclusive;
Image CTFSpectraPolarTrimmed = CTFSpectra.AsPolar((uint)MinFreqInclusive, (uint)(MinFreqInclusive + NFreq));
CTFSpectra.FreeDevice(); // This will only be needed again for the final PS1D.
#region Create background and scale
float[] CurrentScale = _SimulatedScale.Interp(PS1D.Select(p => p.X).ToArray());
Image CTFSpectraScale = new Image(new int3(NFreq, DimsRegion.X, 1));
float[] CTFSpectraScaleData = CTFSpectraScale.GetHost(Intent.Write)[0];
// Trim polar to relevant frequencies, and populate coordinates.
Parallel.For(0, DimsRegion.X, y =>
{
float Angle = ((float)y / DimsRegion.X + 0.5f) * (float)Math.PI;
for (int x = 0; x < NFreq; x++)
CTFSpectraScaleData[y * NFreq + x] = CurrentScale[x + MinFreqInclusive];
});
//CTFSpectraScale.WriteMRC("ctfspectrascale.mrc");
// Background is just 1 line since we're in polar.
Image CurrentBackground = new Image(_SimulatedBackground.Interp(PS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq).ToArray());
#endregion
CTFSpectraPolarTrimmed.SubtractFromLines(CurrentBackground);
CurrentBackground.Dispose();
// Normalize background-subtracted spectra.
GPU.Normalize(CTFSpectraPolarTrimmed.GetDevice(Intent.Read),
CTFSpectraPolarTrimmed.GetDevice(Intent.Write),
(uint)CTFSpectraPolarTrimmed.ElementsSliceReal,
(uint)CTFSpectraGrid.Elements());
//CTFSpectraPolarTrimmed.WriteMRC("ctfspectrapolartrimmed.mrc");
#region Convert to fp16
Image CTFSpectraPolarTrimmedHalf = CTFSpectraPolarTrimmed.AsHalf();
CTFSpectraPolarTrimmed.Dispose();
Image CTFSpectraScaleHalf = CTFSpectraScale.AsHalf();
CTFSpectraScale.Dispose();
Image CTFCoordsPolarTrimmedHalf = CTFCoordsPolarTrimmed.AsHalf();
#endregion
// Wiggle weights show how the defocus on the spectra grid is altered
// by changes in individual anchor points of the spline grid.
// They are used later to compute the dScore/dDefocus values for each spectrum
// only once, and derive the values for each anchor point from them.
float[][] WiggleWeights = GridCTF.GetWiggleWeights(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 1f / (CTFGridZ + 1)));
float[][] WiggleWeightsPhase = GridCTFPhase.GetWiggleWeights(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 1f / (CTFGridZ + 1)));
// Helper method for getting CTFStructs for the entire spectra grid.
Func<double[], CTF, float[], float[], CTFStruct[]> EvalGetCTF = (input, ctf, defocusValues, phaseValues) =>
{
decimal AlteredDelta = (decimal)input[input.Length - 2];
decimal AlteredAngle = (decimal)(input[input.Length - 1] * 20 / (Math.PI / 180));
CTF Local = ctf.GetCopy();
Local.DefocusDelta = AlteredDelta;
Local.DefocusAngle = AlteredAngle;
CTFStruct LocalStruct = Local.ToStruct();
CTFStruct[] LocalParams = new CTFStruct[defocusValues.Length];
for (int i = 0; i < LocalParams.Length; i++)
{
LocalParams[i] = LocalStruct;
LocalParams[i].Defocus = defocusValues[i] * -1e-6f;
LocalParams[i].PhaseShift = phaseValues[i] * (float)Math.PI;
}
return LocalParams;
};
// Simulate with adjusted CTF, compare to originals
#region Eval and Gradient methods
float BorderZ = 0.5f / CTFGridZ;
Func<double[], double> Eval = input =>
{
CubicGrid Altered = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] DefocusValues = Altered.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CubicGrid AlteredPhase = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] PhaseValues = AlteredPhase.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
float[] Result = new float[LocalParams.Length];
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
Result,
(uint)LocalParams.Length);
float Score = 0;
for (int i = 0; i < Result.Length; i++)
if (CTFSpectraConsider[i])
Score += Result[i];
Score /= NCTFSpectraConsider;
if (float.IsNaN(Score) || float.IsInfinity(Score))
throw new Exception("Bad score.");
return (1.0 - Score) * 1000.0;
};
Func<double[], double[]> Gradient = input =>
{
const float Step = 0.005f;
double[] Result = new double[input.Length];
// In 0D grid case, just get gradient for all 4 parameters.
// In 1+D grid case, do simple gradient for astigmatism and phase...
int StartComponent = input.Length - 2;
//int StartComponent = 0;
for (int i = StartComponent; i < input.Length; i++)
{
double[] UpperInput = new double[input.Length];
input.CopyTo(UpperInput, 0);
UpperInput[i] += Step;
double UpperValue = Eval(UpperInput);
double[] LowerInput = new double[input.Length];
input.CopyTo(LowerInput, 0);
LowerInput[i] -= Step;
double LowerValue = Eval(LowerInput);
Result[i] = (UpperValue - LowerValue) / (2f * Step);
}
float[] ResultPlus = new float[CTFSpectraGrid.Elements()];
float[] ResultMinus = new float[CTFSpectraGrid.Elements()];
// ..., take shortcut for defoci...
{
CubicGrid AlteredPhase = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] PhaseValues = AlteredPhase.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
{
CubicGrid AlteredPlus = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v + Step).ToArray());
float[] DefocusValues = AlteredPlus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
ResultPlus,
(uint)LocalParams.Length);
}
{
CubicGrid AlteredMinus = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v - Step).ToArray());
float[] DefocusValues = AlteredMinus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
ResultMinus,
(uint)LocalParams.Length);
}
float[] LocalGradients = new float[ResultPlus.Length];
for (int i = 0; i < LocalGradients.Length; i++)
LocalGradients[i] = ResultMinus[i] - ResultPlus[i];
// Now compute gradients per grid anchor point using the precomputed individual gradients and wiggle factors.
Parallel.For(0, GridCTF.Dimensions.Elements(), i => Result[i] = MathHelper.ReduceWeighted(LocalGradients, WiggleWeights[i]) / LocalGradients.Length / (2f * Step) * 1000f);
}
// ..., and take shortcut for phases.
if (MainWindow.Options.CTFDoPhase)
{
CubicGrid AlteredPlus = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] DefocusValues = AlteredPlus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
{
CubicGrid AlteredPhasePlus = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v + Step).ToArray());
float[] PhaseValues = AlteredPhasePlus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
ResultPlus,
(uint)LocalParams.Length);
}
{
CubicGrid AlteredPhaseMinus = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v - Step).ToArray());
float[] PhaseValues = AlteredPhaseMinus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
ResultMinus,
(uint)LocalParams.Length);
}
float[] LocalGradients = new float[ResultPlus.Length];
for (int i = 0; i < LocalGradients.Length; i++)
LocalGradients[i] = ResultMinus[i] - ResultPlus[i];
// Now compute gradients per grid anchor point using the precomputed individual gradients and wiggle factors.
Parallel.For(0, GridCTFPhase.Dimensions.Elements(), i => Result[i + GridCTF.Dimensions.Elements()] = MathHelper.ReduceWeighted(LocalGradients, WiggleWeightsPhase[i]) / LocalGradients.Length / (2f * Step) * 1000f);
}
foreach (var i in Result)
if (double.IsNaN(i) || double.IsInfinity(i))
throw new Exception("Bad score.");
return Result;
};
#endregion
#region Minimize first time with potential outpiers
double[] StartParams = new double[GridCTF.Dimensions.Elements() + GridCTFPhase.Dimensions.Elements() + 2];
for (int i = 0; i < GridCTF.Dimensions.Elements(); i++)
StartParams[i] = GridCTF.FlatValues[i];
for (int i = 0; i < GridCTFPhase.Dimensions.Elements(); i++)
StartParams[i + GridCTF.Dimensions.Elements()] = GridCTFPhase.FlatValues[i];
StartParams[StartParams.Length - 2] = (double)CTF.DefocusDelta;
StartParams[StartParams.Length - 1] = (double)CTF.DefocusAngle / 20 * (Math.PI / 180);
// Compute correlation for individual spectra, and throw away those that are >.75 sigma worse than mean.
#region Discard outliers
if (CTFSpace || CTFTime)
{
CubicGrid Altered = new CubicGrid(GridCTF.Dimensions, StartParams.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] DefocusValues = Altered.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CubicGrid AlteredPhase = new CubicGrid(GridCTFPhase.Dimensions, StartParams.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] PhaseValues = AlteredPhase.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = EvalGetCTF(StartParams, CTF, DefocusValues, PhaseValues);
float[] Result = new float[LocalParams.Length];
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
Result,
(uint)LocalParams.Length);
float MeanResult = MathHelper.Mean(Result);
float StdResult = MathHelper.StdDev(Result);
CTFSpectraConsider = new bool[CTFSpectraGrid.Elements()];
Parallel.For(0, CTFSpectraConsider.Length, i =>
{
//if (Result[i] > MeanResult - StdResult * 1.5f)
CTFSpectraConsider[i] = true;
/*else
{
CTFSpectraConsider[i] = false;
for (int j = 0; j < WiggleWeights.Length; j++)
// Make sure the spectrum's gradient doesn't affect the overall gradient.
WiggleWeights[j][i] = 0;
}*/
});
NCTFSpectraConsider = CTFSpectraConsider.Where(v => v).Count();
}
#endregion
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Gradient)
{
Past = 1,
Delta = 1e-6,
MaxLineSearch = 15,
Corrections = 20
};
Optimizer.Minimize(StartParams);
#endregion
#region Retrieve parameters
CTF.Defocus = (decimal)MathHelper.Mean(Optimizer.Solution.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v));
CTF.DefocusDelta = (decimal)Optimizer.Solution[StartParams.Length - 2];
CTF.DefocusAngle = (decimal)(Optimizer.Solution[StartParams.Length - 1] * 20 / (Math.PI / 180));
CTF.PhaseShift = (decimal)MathHelper.Mean(Optimizer.Solution.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v));
if (CTF.DefocusDelta < 0)
{
CTF.DefocusAngle += 90;
CTF.DefocusDelta *= -1;
}
CTF.DefocusAngle = ((int)CTF.DefocusAngle + 180 * 99) % 180;
GridCTF = new CubicGrid(GridCTF.Dimensions, Optimizer.Solution.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
GridCTFPhase = new CubicGrid(GridCTFPhase.Dimensions, Optimizer.Solution.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
#endregion
// Dispose GPU resources manually because GC can't be bothered to do it in time.
CTFSpectraPolarTrimmedHalf.Dispose();
CTFCoordsPolarTrimmedHalf.Dispose();
CTFSpectraScaleHalf.Dispose();
#region Get nicer envelope fit
if (preciseFit >= 2)
{
if (!CTFSpace && !CTFTime)
{
UpdateRotationalAverage(true);
}
else
{
Image CTFSpectraBackground = new Image(new int3(DimsRegion), true);
float[] CTFSpectraBackgroundData = CTFSpectraBackground.GetHost(Intent.Write)[0];
// Construct background in Cartesian coordinates.
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) =>
{
CTFSpectraBackgroundData[y * CTFSpectraBackground.DimsEffective.X + x] = _SimulatedBackground.Interp(r / DimsRegion.X);
});
CTFSpectra.SubtractFromSlices(CTFSpectraBackground);
float[] DefocusValues = GridCTF.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, BorderZ));
CTFStruct[] LocalParams = DefocusValues.Select(v =>
{
CTF Local = CTF.GetCopy();
Local.Defocus = (decimal)v + 0.0M;
return Local.ToStruct();
}).ToArray();
Image CTFAverage1D = new Image(IntPtr.Zero, new int3(DimsRegion.X / 2, 1, 1));
CTF CTFAug = CTF.GetCopy();
CTFAug.Defocus += 0.0M;
GPU.CTFMakeAverage(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceReal,
(uint)DimsRegion.X,
LocalParams,
CTFAug.ToStruct(),
0,
(uint)DimsRegion.X / 2,
CTFSpectraConsider.Select(v => v ? 1 : 0).ToArray(),
(uint)CTFSpectraGrid.Elements(),
CTFAverage1D.GetDevice(Intent.Write));
CTFSpectra.AddToSlices(CTFSpectraBackground);
float[] RotationalAverageData = CTFAverage1D.GetHost(Intent.Read)[0];
float2[] ForPS1D = new float2[PS1D.Length];
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, (float)Math.Round(RotationalAverageData[i], 4) + _SimulatedBackground.Interp((float)i / DimsRegion.X));
MathHelper.UnNaN(ForPS1D);
_PS1D = ForPS1D;
CTFSpectraBackground.Dispose();
CTFAverage1D.Dispose();
CTFSpectra.FreeDevice();
}
CTF.Defocus = Math.Max(CTF.Defocus, MainWindow.Options.CTFZMin);
UpdateBackgroundFit();
}
#endregion
}
#endregion
// Subtract background from 2D average and write it to disk.
// This image is used for quick visualization purposes only.
#region PS2D update
{
int3 DimsAverage = new int3(DimsRegion.X, DimsRegion.X / 2, 1);
float[] Average2DData = new float[DimsAverage.Elements()];
float[] OriginalAverageData = CTFMean.GetHost(Intent.Read)[0];
for (int y = 0; y < DimsAverage.Y; y++)
{
int yy = y * y;
for (int x = 0; x < DimsAverage.Y; x++)
{
int xx = DimsRegion.X / 2 - x - 1;
xx *= xx;
float r = (float)Math.Sqrt(xx + yy) / DimsRegion.X;
Average2DData[y * DimsAverage.X + x] = OriginalAverageData[(y + DimsRegion.X / 2) * (DimsRegion.X / 2 + 1) + x] - SimulatedBackground.Interp(r);
}
for (int x = 0; x < DimsRegion.X / 2; x++)
{
int xx = x * x;
float r = (float)Math.Sqrt(xx + yy) / DimsRegion.X;
Average2DData[y * DimsAverage.X + x + DimsRegion.X / 2] = OriginalAverageData[(DimsRegion.X / 2 - y) * (DimsRegion.X / 2 + 1) + (DimsRegion.X / 2 - 1 - x)] - SimulatedBackground.Interp(r);
}
}
IOHelper.WriteMapFloat(PowerSpectrumPath,
new HeaderMRC
{
Dimensions = DimsAverage,
MinValue = MathHelper.Min(Average2DData),
MaxValue = MathHelper.Max(Average2DData)
},
Average2DData);
PS2DTemp = null;
OnPropertyChanged("PS2D");
}
#endregion
for (int i = 0; i < PS1D.Length; i++)
PS1D[i].Y -= SimulatedBackground.Interp(PS1D[i].X);
SimulatedBackground = new Cubic1D(SimulatedBackground.Data.Select(v => new float2(v.X, 0f)).ToArray());
OnPropertyChanged("PS1D");
CTFSpectra.Dispose();
CTFMean.Dispose();
CTFCoordsCart.Dispose();
CTFCoordsPolarTrimmed.Dispose();
Simulated1D = GetSimulated1D();
CTFQuality = GetCTFQuality();
SaveMeta();
}
public void lbfgsTest3()
{
Accord.Math.Tools.SetupGenerator(0);
Func<double[], double> f;
Func<double[], double[]> g;
createExpDiff(out f, out g);
int errors = 0;
for (int i = 0; i < 10000; i++)
{
double[] start = Accord.Math.Matrix.Random(2, -1.0, 1.0);
var lbfgs = new BroydenFletcherGoldfarbShanno(numberOfVariables: 2, function: f, gradient: g);
lbfgs.Minimize(start);
double minValue = lbfgs.Value;
double[] solution = lbfgs.Solution;
double expected = -2;
if (Math.Abs(expected - minValue) > 1e-3)
errors++;
}
Assert.IsTrue(errors < 800);
}
public void MutableGradientSizeTest()
{
BroydenFletcherGoldfarbShanno target = new BroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0,
Gradient = (x) => x
};
target.Minimize();
}
public void WrongGradientSizeTest()
{
BroydenFletcherGoldfarbShanno target = new BroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0,
Gradient = (x) => new double[1]
};
target.Minimize();
}
public void NoGradientTest()
{
BroydenFletcherGoldfarbShanno target = new BroydenFletcherGoldfarbShanno(2)
{
Function = (x) => 0.0
};
target.Minimize();
}
public void ProcessShift(MapHeader originalHeader, Image originalStack, decimal scaleFactor)
{
// Deal with dimensions and grids.
int NFrames = originalHeader.Dimensions.Z;
int2 DimsImage = new int2(originalHeader.Dimensions);
int2 DimsRegion = new int2(768, 768);
float OverlapFraction = 0.0f;
int2 DimsPositionGrid;
int3[] PositionGrid = Helper.GetEqualGridSpacing(DimsImage, DimsRegion, OverlapFraction, out DimsPositionGrid);
//PositionGrid = new[] { new int3(0, 0, 0) };
//DimsPositionGrid = new int2(1, 1);
int NPositions = PositionGrid.Length;
int ShiftGridX = 1;
int ShiftGridY = 1;
int ShiftGridZ = Math.Min(NFrames, MainWindow.Options.GridMoveZ);
GridMovementX = new CubicGrid(new int3(ShiftGridX, ShiftGridY, ShiftGridZ));
GridMovementY = new CubicGrid(new int3(ShiftGridX, ShiftGridY, ShiftGridZ));
int LocalGridX = Math.Min(DimsPositionGrid.X, MainWindow.Options.GridMoveX);
int LocalGridY = Math.Min(DimsPositionGrid.Y, MainWindow.Options.GridMoveY);
int LocalGridZ = Math.Min(2, NFrames);
GridLocalX = new CubicGrid(new int3(LocalGridX, LocalGridY, LocalGridZ));
GridLocalY = new CubicGrid(new int3(LocalGridX, LocalGridY, LocalGridZ));
int3 ShiftGrid = new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames);
int MinFreqInclusive = (int)(MainWindow.Options.MovementRangeMin * DimsRegion.X / 2);
int MaxFreqExclusive = (int)(MainWindow.Options.MovementRangeMax * DimsRegion.X / 2);
int NFreq = MaxFreqExclusive - MinFreqInclusive;
int CentralFrame = NFrames / 2;
int MaskExpansions = Math.Max(1, ShiftGridZ / 3);
int[] MaskSizes = new int[MaskExpansions];
// Allocate memory and create all prerequisites:
int MaskLength;
Image ShiftFactors;
Image Phases;
Image PhasesAverage;
Image Shifts;
{
List<long> Positions = new List<long>();
List<float2> Factors = new List<float2>();
List<float2> Freq = new List<float2>();
int Min2 = MinFreqInclusive * MinFreqInclusive;
int Max2 = MaxFreqExclusive * MaxFreqExclusive;
float PixelSize = (float)(MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5f;
float PixelDelta = (float)(MainWindow.Options.CTFPixelMax - MainWindow.Options.CTFPixelMin) * 0.5f;
float PixelAngle = (float)MainWindow.Options.CTFPixelAngle;
for (int y = 0; y < DimsRegion.Y; y++)
{
int yy = y - DimsRegion.X / 2;
for (int x = 0; x < DimsRegion.X / 2 + 1; x++)
{
int xx = x - DimsRegion.X / 2;
int r2 = xx * xx + yy * yy;
if (r2 >= Min2 && r2 < Max2)
{
Positions.Add(y * (DimsRegion.X / 2 + 1) + x);
Factors.Add(new float2((float)xx / DimsRegion.X * 2f * (float)Math.PI,
(float)yy / DimsRegion.X * 2f * (float)Math.PI));
float Angle = (float)Math.Atan2(yy, xx);
float r = (float)Math.Sqrt(r2);
Freq.Add(new float2(r, Angle));
}
}
}
// Sort everyone with ascending distance from center.
List<KeyValuePair<float, int>> FreqIndices = Freq.Select((v, i) => new KeyValuePair<float, int>(v.X, i)).ToList();
FreqIndices.Sort((a, b) => a.Key.CompareTo(b.Key));
int[] SortedIndices = FreqIndices.Select(v => v.Value).ToArray();
Helper.Reorder(Positions, SortedIndices);
Helper.Reorder(Factors, SortedIndices);
Helper.Reorder(Freq, SortedIndices);
float Bfac = (float)MainWindow.Options.MovementBfactor * 0.25f / PixelSize / DimsRegion.X;
float2[] BfacWeightsData = Freq.Select(v => (float)Math.Exp(v.X * Bfac)).Select(v => new float2(v, v)).ToArray();
Image BfacWeights = new Image(Helper.ToInterleaved(BfacWeightsData), false, false, false);
long[] RelevantMask = Positions.ToArray();
ShiftFactors = new Image(Helper.ToInterleaved(Factors.ToArray()));
MaskLength = RelevantMask.Length;
// Get mask sizes for different expansion steps.
for (int i = 0; i < MaskExpansions; i++)
{
float CurrentMaxFreq = MinFreqInclusive + (MaxFreqExclusive - MinFreqInclusive) / (float)MaskExpansions * (i + 1);
MaskSizes[i] = Freq.Count(v => v.X * v.X < CurrentMaxFreq * CurrentMaxFreq);
}
Phases = new Image(IntPtr.Zero, new int3(MaskLength * 2, DimsPositionGrid.X * DimsPositionGrid.Y, NFrames), false, false, false);
GPU.CreateShift(originalStack.GetDevice(Intent.Read),
new int2(originalHeader.Dimensions),
originalHeader.Dimensions.Z,
PositionGrid,
PositionGrid.Length,
DimsRegion,
RelevantMask,
(uint)MaskLength,
Phases.GetDevice(Intent.Write));
Phases.MultiplyLines(BfacWeights);
BfacWeights.Dispose();
originalStack.FreeDevice();
PhasesAverage = new Image(IntPtr.Zero, new int3(MaskLength, NPositions, 1), false, true, false);
Shifts = new Image(new float[NPositions * NFrames * 2]);
}
#region Fit global movement
{
int MinXSteps = 1, MinYSteps = 1;
int MinZSteps = Math.Min(NFrames, 3);
int3 ExpansionGridSize = new int3(MinXSteps, MinYSteps, MinZSteps);
float[][] WiggleWeights = new CubicGrid(ExpansionGridSize).GetWiggleWeights(ShiftGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
double[] StartParams = new double[ExpansionGridSize.Elements() * 2];
for (int m = 0; m < MaskExpansions; m++)
{
double[] LastAverage = null;
Action<double[]> SetPositions = input =>
{
// Construct CubicGrids and get interpolated shift values.
CubicGrid AlteredGridX = new CubicGrid(ExpansionGridSize, input.Where((v, i) => i % 2 == 0).Select(v => (float)v).ToArray());
float[] AlteredX = AlteredGridX.GetInterpolatedNative(new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames),
new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
CubicGrid AlteredGridY = new CubicGrid(ExpansionGridSize, input.Where((v, i) => i % 2 == 1).Select(v => (float)v).ToArray());
float[] AlteredY = AlteredGridY.GetInterpolatedNative(new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames),
new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
// Let movement start at 0 in the central frame.
/*float2[] CenterFrameOffsets = new float2[NPositions];
for (int i = 0; i < NPositions; i++)
CenterFrameOffsets[i] = new float2(AlteredX[CentralFrame * NPositions + i], AlteredY[CentralFrame * NPositions + i]);*/
// Finally, set the shift values in the device array.
float[] ShiftData = Shifts.GetHost(Intent.Write)[0];
Parallel.For(0, AlteredX.Length, i =>
{
ShiftData[i * 2] = AlteredX[i];// - CenterFrameOffsets[i % NPositions].X;
ShiftData[i * 2 + 1] = AlteredY[i];// - CenterFrameOffsets[i % NPositions].Y;
});
};
Action<double[]> DoAverage = input =>
{
if (LastAverage == null || input.Where((t, i) => t != LastAverage[i]).Any())
{
SetPositions(input);
GPU.ShiftGetAverage(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Write),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
(uint)NPositions,
(uint)NFrames);
if (LastAverage == null)
LastAverage = new double[input.Length];
Array.Copy(input, LastAverage, input.Length);
}
};
Func<double[], double> Eval = input =>
{
DoAverage(input);
float[] Diff = new float[NPositions * NFrames];
GPU.ShiftGetDiff(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
Diff,
(uint)NPositions,
(uint)NFrames);
for (int i = 0; i < Diff.Length; i++)
Diff[i] = Diff[i];// * 100f;
return Diff.Sum();
};
Func<double[], double[]> Grad = input =>
{
DoAverage(input);
float[] GradX = new float[NPositions * NFrames], GradY = new float[NPositions * NFrames];
float[] Diff = new float[NPositions * NFrames * 2];
GPU.ShiftGetGrad(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
Diff,
(uint)NPositions,
(uint)NFrames);
//for (int i = 0; i < Diff.Length; i++)
//Diff[i] = Diff[i] * 100f;
for (int i = 0; i < GradX.Length; i++)
{
GradX[i] = Diff[i * 2];
GradY[i] = Diff[i * 2 + 1];
}
double[] Result = new double[input.Length];
Parallel.For(0, input.Length / 2, i =>
{
Result[i * 2] = MathHelper.ReduceWeighted(GradX, WiggleWeights[i]);
Result[i * 2 + 1] = MathHelper.ReduceWeighted(GradY, WiggleWeights[i]);
});
return Result;
};
/*Func<double[], double[]> Grad = input =>
{
DoAverage(input);
float[] GradX = new float[NPositions * NFrames], GradY = new float[NPositions * NFrames];
float Step = 0.002f;
{
double[] InputXP = new double[input.Length];
for (int i = 0; i < input.Length; i++)
if (i % 2 == 0)
InputXP[i] = input[i] + Step;
else
InputXP[i] = input[i];
SetPositions(InputXP);
float[] DiffXP = new float[NPositions * NFrames];
GPU.ShiftGetDiff(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
DiffXP,
(uint)NPositions,
(uint)NFrames);
double[] InputXM = new double[input.Length];
for (int i = 0; i < input.Length; i++)
if (i % 2 == 0)
InputXM[i] = input[i] - Step;
else
InputXM[i] = input[i];
SetPositions(InputXM);
float[] DiffXM = new float[NPositions * NFrames];
GPU.ShiftGetDiff(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
DiffXM,
(uint)NPositions,
(uint)NFrames);
for (int i = 0; i < GradX.Length; i++)
GradX[i] = (DiffXP[i] - DiffXM[i]) / (Step * 2);
}
{
double[] InputYP = new double[input.Length];
for (int i = 0; i < input.Length; i++)
if (i % 2 == 1)
InputYP[i] = input[i] + Step;
else
InputYP[i] = input[i];
SetPositions(InputYP);
float[] DiffYP = new float[NPositions * NFrames];
GPU.ShiftGetDiff(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
DiffYP,
(uint)NPositions,
(uint)NFrames);
double[] InputYM = new double[input.Length];
for (int i = 0; i < input.Length; i++)
if (i % 2 == 1)
InputYM[i] = input[i] - Step;
else
InputYM[i] = input[i];
SetPositions(InputYM);
float[] DiffYM = new float[NPositions * NFrames];
GPU.ShiftGetDiff(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
DiffYM,
(uint)NPositions,
(uint)NFrames);
for (int i = 0; i < GradY.Length; i++)
GradY[i] = (DiffYP[i] - DiffYM[i]) / (Step * 2);
}
double[] Result = new double[input.Length];
Parallel.For(0, input.Length / 2, i =>
{
Result[i * 2] = MathHelper.ReduceWeighted(GradX, WiggleWeights[i]);
Result[i * 2 + 1] = MathHelper.ReduceWeighted(GradY, WiggleWeights[i]);
});
return Result;
};*/
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
Optimizer.Corrections = 20;
Optimizer.Minimize(StartParams);
float MeanX = MathHelper.Mean(Optimizer.Solution.Where((v, i) => i % 2 == 0).Select(v => (float)v));
float MeanY = MathHelper.Mean(Optimizer.Solution.Where((v, i) => i % 2 == 1).Select(v => (float)v));
for (int i = 0; i < ExpansionGridSize.Elements(); i++)
{
Optimizer.Solution[i * 2] -= MeanX;
Optimizer.Solution[i * 2 + 1] -= MeanY;
}
// Store coarse values in grids.
GridMovementX = new CubicGrid(ExpansionGridSize, Optimizer.Solution.Where((v, i) => i % 2 == 0).Select(v => (float)v).ToArray());
GridMovementY = new CubicGrid(ExpansionGridSize, Optimizer.Solution.Where((v, i) => i % 2 == 1).Select(v => (float)v).ToArray());
if (m < MaskExpansions - 1)
{
// Refine sampling.
ExpansionGridSize = new int3((int)Math.Round((float)(ShiftGridX - MinXSteps) / (MaskExpansions - 1) * (m + 1) + MinXSteps),
(int)Math.Round((float)(ShiftGridY - MinYSteps) / (MaskExpansions - 1) * (m + 1) + MinYSteps),
(int)Math.Round((float)(ShiftGridZ - MinZSteps) / (MaskExpansions - 1) * (m + 1) + MinZSteps));
WiggleWeights = new CubicGrid(ExpansionGridSize).GetWiggleWeights(ShiftGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
// Resize the grids to account for finer sampling.
GridMovementX = GridMovementX.Resize(ExpansionGridSize);
GridMovementY = GridMovementY.Resize(ExpansionGridSize);
// Construct start parameters for next optimization iteration.
StartParams = new double[ExpansionGridSize.Elements() * 2];
for (int i = 0; i < ExpansionGridSize.Elements(); i++)
{
StartParams[i * 2] = GridMovementX.FlatValues[i];
StartParams[i * 2 + 1] = GridMovementY.FlatValues[i];
}
}
}
}
#endregion
// Center the global shifts
/*{
float2[] AverageShifts = new float2[ShiftGridZ];
for (int i = 0; i < AverageShifts.Length; i++)
AverageShifts[i] = new float2(MathHelper.Mean(GridMovementX.GetSliceXY(i)),
MathHelper.Mean(GridMovementY.GetSliceXY(i)));
float2 CenterShift = MathHelper.Mean(AverageShifts);
GridMovementX = new CubicGrid(GridMovementX.Dimensions, GridMovementX.FlatValues.Select(v => v - CenterShift.X).ToArray());
GridMovementY = new CubicGrid(GridMovementY.Dimensions, GridMovementY.FlatValues.Select(v => v - CenterShift.Y).ToArray());
}*/
#region Fit local movement
/*{
int MinXSteps = LocalGridX, MinYSteps = LocalGridY;
int MinZSteps = LocalGridZ;
int3 ExpansionGridSize = new int3(MinXSteps, MinYSteps, MinZSteps);
float[][] WiggleWeights = new CubicGrid(ExpansionGridSize).GetWiggleWeights(ShiftGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
double[] StartParams = new double[ExpansionGridSize.Elements() * 2];
for (int m = MaskExpansions - 1; m < MaskExpansions; m++)
{
double[] LastAverage = null;
Action<double[]> SetPositions = input =>
{
// Construct CubicGrids and get interpolated shift values.
float[] GlobalX = GridMovementX.GetInterpolatedNative(new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames),
new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
CubicGrid AlteredGridX = new CubicGrid(ExpansionGridSize, input.Where((v, i) => i % 2 == 0).Select(v => (float)v).ToArray());
float[] AlteredX = AlteredGridX.GetInterpolatedNative(new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames),
new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
AlteredX = MathHelper.Plus(GlobalX, AlteredX);
float[] GlobalY = GridMovementY.GetInterpolatedNative(new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames),
new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
CubicGrid AlteredGridY = new CubicGrid(ExpansionGridSize, input.Where((v, i) => i % 2 == 1).Select(v => (float)v).ToArray());
float[] AlteredY = AlteredGridY.GetInterpolatedNative(new int3(DimsPositionGrid.X, DimsPositionGrid.Y, NFrames),
new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
AlteredY = MathHelper.Plus(GlobalY, AlteredY);
// Let movement start at 0 in the central frame.
float2[] CenterFrameOffsets = new float2[NPositions];
for (int i = 0; i < NPositions; i++)
CenterFrameOffsets[i] = new float2(AlteredX[CentralFrame * NPositions + i], AlteredY[CentralFrame * NPositions + i]);
// Finally, set the shift values in the device array.
float[] ShiftData = Shifts.GetHost(Intent.Write)[0];
Parallel.For(0, AlteredX.Length, i =>
{
ShiftData[i * 2] = AlteredX[i] - CenterFrameOffsets[i % NPositions].X;
ShiftData[i * 2 + 1] = AlteredY[i] - CenterFrameOffsets[i % NPositions].Y;
});
};
Action<double[]> DoAverage = input =>
{
if (LastAverage == null || input.Where((t, i) => t != LastAverage[i]).Any())
{
SetPositions(input);
GPU.ShiftGetAverage(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Write),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
(uint)NPositions,
(uint)NFrames);
if (LastAverage == null)
LastAverage = new double[input.Length];
Array.Copy(input, LastAverage, input.Length);
}
};
Func<double[], double> Eval = input =>
{
DoAverage(input);
float[] Diff = new float[NPositions * NFrames];
GPU.ShiftGetDiff(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
Diff,
(uint)NPositions,
(uint)NFrames);
for (int i = 0; i < Diff.Length; i++)
Diff[i] = Diff[i] * 100f;
return MathHelper.Mean(Diff);
};
Func<double[], double[]> Grad = input =>
{
DoAverage(input);
float[] Diff = new float[NPositions * NFrames * 2];
GPU.ShiftGetGrad(Phases.GetDevice(Intent.Read),
PhasesAverage.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
Diff,
(uint)NPositions,
(uint)NFrames);
for (int i = 0; i < Diff.Length; i++)
Diff[i] = Diff[i] * 100f;
float[] DiffX = new float[NPositions * NFrames], DiffY = new float[NPositions * NFrames];
for (int i = 0; i < DiffX.Length; i++)
{
DiffX[i] = Diff[i * 2];
DiffY[i] = Diff[i * 2 + 1];
}
double[] Result = new double[input.Length];
Parallel.For(0, input.Length / 2, i =>
{
Result[i * 2] = MathHelper.ReduceWeighted(DiffX, WiggleWeights[i]);
Result[i * 2 + 1] = MathHelper.ReduceWeighted(DiffY, WiggleWeights[i]);
});
return Result;
};
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
Optimizer.Corrections = 20;
Optimizer.Minimize(StartParams);
float MeanX = MathHelper.Mean(Optimizer.Solution.Where((v, i) => i % 2 == 0).Select(v => (float)v));
float MeanY = MathHelper.Mean(Optimizer.Solution.Where((v, i) => i % 2 == 1).Select(v => (float)v));
for (int i = 0; i < ExpansionGridSize.Elements(); i++)
{
Optimizer.Solution[i * 2] -= MeanX;
Optimizer.Solution[i * 2 + 1] -= MeanY;
}
// Store coarse values in grids.
GridLocalX = new CubicGrid(ExpansionGridSize, Optimizer.Solution.Where((v, i) => i % 2 == 0).Select(v => (float)v).ToArray());
GridLocalY = new CubicGrid(ExpansionGridSize, Optimizer.Solution.Where((v, i) => i % 2 == 1).Select(v => (float)v).ToArray());
if (m < MaskExpansions - 1)
{
// Refine sampling.
ExpansionGridSize = new int3((int)Math.Round((float)(LocalGridX - MinXSteps) / (MaskExpansions - 1) * (m + 1) + MinXSteps),
(int)Math.Round((float)(LocalGridY - MinYSteps) / (MaskExpansions - 1) * (m + 1) + MinYSteps),
(int)Math.Round((float)(LocalGridZ - MinZSteps) / (MaskExpansions - 1) * (m + 1) + MinZSteps));
WiggleWeights = new CubicGrid(ExpansionGridSize).GetWiggleWeights(ShiftGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0f));
// Resize the grids to account for finer sampling.
GridLocalX = GridLocalX.Resize(ExpansionGridSize);
GridLocalY = GridLocalY.Resize(ExpansionGridSize);
// Construct start parameters for next optimization iteration.
StartParams = new double[ExpansionGridSize.Elements() * 2];
for (int i = 0; i < ExpansionGridSize.Elements(); i++)
{
StartParams[i * 2] = GridLocalX.FlatValues[i];
StartParams[i * 2 + 1] = GridLocalY.FlatValues[i];
}
}
}
}*/
#endregion
ShiftFactors.Dispose();
Phases.Dispose();
PhasesAverage.Dispose();
Shifts.Dispose();
// Center the local shifts
/*{
float2[] AverageShifts = new float2[LocalGridZ];
for (int i = 0; i < AverageShifts.Length; i++)
AverageShifts[i] = new float2(MathHelper.Mean(GridLocalX.GetSliceXY(i)),
MathHelper.Mean(GridLocalY.GetSliceXY(i)));
float2 CenterShift = MathHelper.Mean(AverageShifts);
GridLocalX = new CubicGrid(GridLocalX.Dimensions, GridLocalX.FlatValues.Select(v => v - CenterShift.X).ToArray());
GridLocalY = new CubicGrid(GridLocalY.Dimensions, GridLocalY.FlatValues.Select(v => v - CenterShift.Y).ToArray());
}*/
SaveMeta();
}
public void InvalidLineSearchTest2()
{
int n = 10;
Func<double[], double> function = (parameters) =>
{
return -(n * Math.Log(0) - n * Math.Log(Math.PI));
};
Func<double[], double[]> gradient = (parameters) =>
{
return new[] { 2.0, Double.NegativeInfinity };
};
double[] start = { 0, 0 };
var lbfgs = new BroydenFletcherGoldfarbShanno(2, function, gradient);
bool success = lbfgs.Minimize(start);
Assert.IsFalse(success);
}
public void ExportParticlesMovie(Star tableIn, Star tableOut, MapHeader originalHeader, Image originalStack, int size, float particleradius, decimal scaleFactor)
{
int CurrentDevice = GPU.GetDevice();
#region Make sure directories exist.
lock (tableIn)
{
if (!Directory.Exists(ParticleMoviesDir))
Directory.CreateDirectory(ParticleMoviesDir);
if (!Directory.Exists(ParticleCTFMoviesDir))
Directory.CreateDirectory(ParticleCTFMoviesDir);
}
#endregion
#region Get row indices for all, and individual halves
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName))
RowIndices.Add(i);
//RowIndices = RowIndices.Take(13).ToList();
List<int> RowIndices1 = new List<int>();
List<int> RowIndices2 = new List<int>();
for (int i = 0; i < RowIndices.Count; i++)
if (tableIn.GetRowValue(RowIndices[i], "rlnRandomSubset") == "1")
RowIndices1.Add(RowIndices[i]);
else
RowIndices2.Add(RowIndices[i]);
#endregion
if (RowIndices.Count == 0)
return;
#region Auxiliary variables
List<int> TableOutIndices = new List<int>();
int3 Dims = originalHeader.Dimensions;
Dims.Z = 36;
int3 DimsRegion = new int3(size, size, 1);
int3 DimsPadded = new int3(size * 2, size * 2, 1);
int NParticles = RowIndices.Count;
int NParticles1 = RowIndices1.Count;
int NParticles2 = RowIndices2.Count;
float PixelSize = (float)CTF.PixelSize / 1.00f;
float PixelDelta = (float)CTF.PixelSizeDelta / 1.00f;
float PixelAngle = (float)CTF.PixelSizeAngle * Helper.ToRad;
#endregion
#region Prepare initial coordinates and shifts
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
int3[] Origins1 = new int3[NParticles1];
int3[] Origins2 = new int3[NParticles2];
float3[] ResidualShifts1 = new float3[NParticles1];
float3[] ResidualShifts2 = new float3[NParticles2];
lock (tableIn) // Writing to the table, better be on the safe side
{
// Half1: Add translational shifts to coordinates, sans the fractional part
for (int i = 0; i < NParticles1; i++)
{
float2 Pos = new float2(float.Parse(ColumnPosX[RowIndices1[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices1[i]], CultureInfo.InvariantCulture)) * 1.00f;
float2 Shift = new float2(float.Parse(ColumnOriginX[RowIndices1[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices1[i]], CultureInfo.InvariantCulture)) * 1.00f;
Origins1[i] = new int3((int)(Pos.X - Shift.X),
(int)(Pos.Y - Shift.Y),
0);
ResidualShifts1[i] = new float3(-MathHelper.ResidualFraction(Pos.X - Shift.X),
-MathHelper.ResidualFraction(Pos.Y - Shift.Y),
0f);
tableIn.SetRowValue(RowIndices1[i], "rlnCoordinateX", Origins1[i].X.ToString());
tableIn.SetRowValue(RowIndices1[i], "rlnCoordinateY", Origins1[i].Y.ToString());
tableIn.SetRowValue(RowIndices1[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices1[i], "rlnOriginY", "0.0");
}
// Half2: Add translational shifts to coordinates, sans the fractional part
for (int i = 0; i < NParticles2; i++)
{
float2 Pos = new float2(float.Parse(ColumnPosX[RowIndices2[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices2[i]], CultureInfo.InvariantCulture)) * 1.00f;
float2 Shift = new float2(float.Parse(ColumnOriginX[RowIndices2[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices2[i]], CultureInfo.InvariantCulture)) * 1.00f;
Origins2[i] = new int3((int)(Pos.X - Shift.X),
(int)(Pos.Y - Shift.Y),
0);
ResidualShifts2[i] = new float3(-MathHelper.ResidualFraction(Pos.X - Shift.X),
-MathHelper.ResidualFraction(Pos.Y - Shift.Y),
0f);
tableIn.SetRowValue(RowIndices2[i], "rlnCoordinateX", Origins2[i].X.ToString());
tableIn.SetRowValue(RowIndices2[i], "rlnCoordinateY", Origins2[i].Y.ToString());
tableIn.SetRowValue(RowIndices2[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices2[i], "rlnOriginY", "0.0");
}
}
#endregion
#region Allocate memory for particle and PS stacks
Image ParticleStackAll = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles * Dims.Z));
Image ParticleStack1 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles1 * Dims.Z));
Image ParticleStack2 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles2 * Dims.Z));
Image PSStackAll = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles * Dims.Z), true);
Image PSStack1 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles1 * Dims.Z), true);
Image PSStack2 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles2 * Dims.Z), true);
Image FrameParticles1 = new Image(IntPtr.Zero, new int3(DimsPadded.X, DimsPadded.Y, NParticles1));
Image FrameParticles2 = new Image(IntPtr.Zero, new int3(DimsPadded.X, DimsPadded.Y, NParticles2));
float[][] ParticleStackData = ParticleStackAll.GetHost(Intent.Write);
float[][] ParticleStackData1 = ParticleStack1.GetHost(Intent.Write);
float[][] ParticleStackData2 = ParticleStack2.GetHost(Intent.Write);
float[][] PSStackData = PSStackAll.GetHost(Intent.Write);
float[][] PSStackData1 = PSStack1.GetHost(Intent.Write);
float[][] PSStackData2 = PSStack2.GetHost(Intent.Write);
#endregion
#region Create rows in outTable
lock (tableOut) // Creating rows in outTable, this absolutely needs to be staged sequentially
{
for (int z = 0; z < Dims.Z; z++)
{
for (int i = 0; i < NParticles; i++)
{
int Index = i < NParticles1 ? RowIndices1[i] : RowIndices2[i - NParticles1];
string OriParticlePath = (i + 1).ToString("D6") + "@particles/" + RootName + "_particles.mrcs";
string ParticleName = (z * NParticles + i + 1).ToString("D6") + "@particlemovies/" + RootName + "_particles.mrcs";
string ParticleCTFName = (z * NParticles + i + 1).ToString("D6") + "@particlectfmovies/" + RootName + "_particlectf.mrcs";
List<string> NewRow = tableIn.GetRow(Index).Select(v => v).ToList(); // Get copy of original row.
NewRow[tableOut.GetColumnIndex("rlnOriginalParticleName")] = OriParticlePath;
NewRow[tableOut.GetColumnIndex("rlnAngleRotPrior")] = tableIn.GetRowValue(Index, "rlnAngleRot");
NewRow[tableOut.GetColumnIndex("rlnAngleTiltPrior")] = tableIn.GetRowValue(Index, "rlnAngleTilt");
NewRow[tableOut.GetColumnIndex("rlnAnglePsiPrior")] = tableIn.GetRowValue(Index, "rlnAnglePsi");
NewRow[tableOut.GetColumnIndex("rlnOriginXPrior")] = "0.0";
NewRow[tableOut.GetColumnIndex("rlnOriginYPrior")] = "0.0";
NewRow[tableOut.GetColumnIndex("rlnImageName")] = ParticleName;
NewRow[tableOut.GetColumnIndex("rlnCtfImage")] = ParticleCTFName;
NewRow[tableOut.GetColumnIndex("rlnMicrographName")] = (z + 1).ToString("D6") + "@stack/" + RootName + "_movie.mrcs";
TableOutIndices.Add(tableOut.RowCount);
tableOut.AddRow(NewRow);
}
}
}
#endregion
#region For every frame, extract particles from each half; shift, correct, and norm them
float StepZ = 1f / Math.Max(Dims.Z - 1, 1);
for (int z = 0; z < Dims.Z; z++)
{
float CoordZ = z * StepZ;
#region Extract, correct, and norm particles
#region Half 1
{
if (originalStack != null)
GPU.Extract(originalStack.GetDeviceSlice(z, Intent.Read),
FrameParticles1.GetDevice(Intent.Write),
Dims.Slice(),
DimsPadded,
Helper.ToInterleaved(Origins1.Select(v => new int3(v.X - DimsPadded.X / 2, v.Y - DimsPadded.Y / 2, 0)).ToArray()),
(uint)NParticles1);
// Shift particles
{
float3[] Shifts = new float3[NParticles1];
for (int i = 0; i < NParticles1; i++)
{
float3 Coords = new float3((float)Origins1[i].X / Dims.X, (float)Origins1[i].Y / Dims.Y, CoordZ);
Shifts[i] = ResidualShifts1[i] + new float3(GetShiftFromPyramid(Coords)) * 1.00f;
}
FrameParticles1.ShiftSlices(Shifts);
}
Image FrameParticlesCropped = FrameParticles1.AsPadded(new int2(DimsRegion));
Image FrameParticlesCorrected = FrameParticlesCropped.AsAnisotropyCorrected(new int2(DimsRegion),
PixelSize + PixelDelta / 2f,
PixelSize - PixelDelta / 2f,
PixelAngle,
6);
FrameParticlesCropped.Dispose();
GPU.NormParticles(FrameParticlesCorrected.GetDevice(Intent.Read),
FrameParticlesCorrected.GetDevice(Intent.Write),
DimsRegion,
(uint)(particleradius / PixelSize),
true,
(uint)NParticles1);
float[][] FrameParticlesCorrectedData = FrameParticlesCorrected.GetHost(Intent.Read);
for (int n = 0; n < NParticles1; n++)
{
ParticleStackData[z * NParticles + n] = FrameParticlesCorrectedData[n];
ParticleStackData1[z * NParticles1 + n] = FrameParticlesCorrectedData[n];
}
//FrameParticlesCorrected.WriteMRC("intermediate_particles1.mrc");
FrameParticlesCorrected.Dispose();
}
#endregion
#region Half 2
{
if (originalStack != null)
GPU.Extract(originalStack.GetDeviceSlice(z, Intent.Read),
FrameParticles2.GetDevice(Intent.Write),
Dims.Slice(),
DimsPadded,
Helper.ToInterleaved(Origins2.Select(v => new int3(v.X - DimsPadded.X / 2, v.Y - DimsPadded.Y / 2, 0)).ToArray()),
(uint)NParticles2);
// Shift particles
{
float3[] Shifts = new float3[NParticles2];
for (int i = 0; i < NParticles2; i++)
{
float3 Coords = new float3((float)Origins2[i].X / Dims.X, (float)Origins2[i].Y / Dims.Y, CoordZ);
Shifts[i] = ResidualShifts2[i] + new float3(GetShiftFromPyramid(Coords)) * 1.00f;
}
FrameParticles2.ShiftSlices(Shifts);
}
Image FrameParticlesCropped = FrameParticles2.AsPadded(new int2(DimsRegion));
Image FrameParticlesCorrected = FrameParticlesCropped.AsAnisotropyCorrected(new int2(DimsRegion),
PixelSize + PixelDelta / 2f,
PixelSize - PixelDelta / 2f,
PixelAngle,
6);
FrameParticlesCropped.Dispose();
GPU.NormParticles(FrameParticlesCorrected.GetDevice(Intent.Read),
FrameParticlesCorrected.GetDevice(Intent.Write),
DimsRegion,
(uint)(particleradius / PixelSize),
true,
(uint)NParticles2);
float[][] FrameParticlesCorrectedData = FrameParticlesCorrected.GetHost(Intent.Read);
for (int n = 0; n < NParticles2; n++)
{
ParticleStackData[z * NParticles + NParticles1 + n] = FrameParticlesCorrectedData[n];
ParticleStackData2[z * NParticles2 + n] = FrameParticlesCorrectedData[n];
}
//FrameParticlesCorrected.WriteMRC("intermediate_particles2.mrc");
FrameParticlesCorrected.Dispose();
}
#endregion
#endregion
#region PS Half 1
{
Image PS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles1), true);
PS.Fill(1f);
// Apply motion blur filter.
#region Motion blur weighting
{
const int Samples = 11;
float StartZ = (z - 0.5f) * StepZ;
float StopZ = (z + 0.5f) * StepZ;
float2[] Shifts = new float2[Samples * NParticles1];
for (int p = 0; p < NParticles1; p++)
{
float NormX = (float)Origins1[p].X / Dims.X;
float NormY = (float)Origins1[p].Y / Dims.Y;
for (int zz = 0; zz < Samples; zz++)
{
float zp = StartZ + (StopZ - StartZ) / (Samples - 1) * zz;
float3 Coords = new float3(NormX, NormY, zp);
Shifts[p * Samples + zz] = GetShiftFromPyramid(Coords) * 1.00f;
}
}
Image MotionFilter = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles1), true);
GPU.CreateMotionBlur(MotionFilter.GetDevice(Intent.Write),
DimsRegion,
Helper.ToInterleaved(Shifts.Select(v => new float3(v.X, v.Y, 0)).ToArray()),
Samples,
(uint)NParticles1);
PS.Multiply(MotionFilter);
//MotionFilter.WriteMRC("motion.mrc");
MotionFilter.Dispose();
}
#endregion
float[][] PSData = PS.GetHost(Intent.Read);
for (int n = 0; n < NParticles1; n++)
PSStackData[z * NParticles + n] = PSData[n];
//PS.WriteMRC("intermediate_ps1.mrc");
PS.Dispose();
}
#endregion
#region PS Half 2
{
Image PS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles2), true);
PS.Fill(1f);
// Apply motion blur filter.
#region Motion blur weighting
{
const int Samples = 11;
float StartZ = (z - 0.5f) * StepZ;
float StopZ = (z + 0.5f) * StepZ;
float2[] Shifts = new float2[Samples * NParticles2];
for (int p = 0; p < NParticles2; p++)
{
float NormX = (float)Origins2[p].X / Dims.X;
float NormY = (float)Origins2[p].Y / Dims.Y;
for (int zz = 0; zz < Samples; zz++)
{
float zp = StartZ + (StopZ - StartZ) / (Samples - 1) * zz;
float3 Coords = new float3(NormX, NormY, zp);
Shifts[p * Samples + zz] = GetShiftFromPyramid(Coords) * 1.00f;
}
}
Image MotionFilter = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles2), true);
GPU.CreateMotionBlur(MotionFilter.GetDevice(Intent.Write),
DimsRegion,
Helper.ToInterleaved(Shifts.Select(v => new float3(v.X, v.Y, 0)).ToArray()),
Samples,
(uint)NParticles2);
PS.Multiply(MotionFilter);
//MotionFilter.WriteMRC("motion.mrc");
MotionFilter.Dispose();
}
#endregion
float[][] PSData = PS.GetHost(Intent.Read);
for (int n = 0; n < NParticles2; n++)
PSStackData[z * NParticles + NParticles1 + n] = PSData[n];
//PS.WriteMRC("intermediate_ps2.mrc");
PS.Dispose();
}
#endregion
}
FrameParticles1.Dispose();
FrameParticles2.Dispose();
originalStack.FreeDevice();
#endregion
HeaderMRC ParticlesHeader = new HeaderMRC
{
Pixelsize = new float3(PixelSize, PixelSize, PixelSize)
};
// Do translation and rotation BFGS per particle
{
float MaxHigh = 2.6f;
CubicGrid GridX = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridY = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridRot = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridTilt = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridPsi = new CubicGrid(new int3(NParticles1, 1, 2));
int2 DimsCropped = new int2(DimsRegion / (MaxHigh / PixelSize / 2f)) / 2 * 2;
#region Get coordinates for CTF and Fourier-space shifts
Image CTFCoords;
Image ShiftFactors;
{
float2[] CTFCoordsData = new float2[(DimsCropped.X / 2 + 1) * DimsCropped.Y];
float2[] ShiftFactorsData = new float2[(DimsCropped.X / 2 + 1) * DimsCropped.Y];
for (int y = 0; y < DimsCropped.Y; y++)
for (int x = 0; x < DimsCropped.X / 2 + 1; x++)
{
int xx = x;
int yy = y < DimsCropped.Y / 2 + 1 ? y : y - DimsCropped.Y;
float xs = xx / (float)DimsRegion.X;
float ys = yy / (float)DimsRegion.Y;
float r = (float)Math.Sqrt(xs * xs + ys * ys);
float angle = (float)(Math.Atan2(yy, xx));
CTFCoordsData[y * (DimsCropped.X / 2 + 1) + x] = new float2(r / PixelSize, angle);
ShiftFactorsData[y * (DimsCropped.X / 2 + 1) + x] = new float2((float)-xx / DimsRegion.X * 2f * (float)Math.PI,
(float)-yy / DimsRegion.X * 2f * (float)Math.PI);
}
CTFCoords = new Image(CTFCoordsData, new int3(DimsCropped), true);
ShiftFactors = new Image(ShiftFactorsData, new int3(DimsCropped), true);
}
#endregion
#region Get inverse sigma2 spectrum for this micrograph from Relion's model.star
Image Sigma2Noise = new Image(new int3(DimsCropped), true);
{
int GroupNumber = int.Parse(tableIn.GetRowValue(RowIndices[0], "rlnGroupNumber"));
//Star SigmaTable = new Star("D:\\rado27\\Refine3D\\run1_ct5_it009_half1_model.star", "data_model_group_" + GroupNumber);
Star SigmaTable = new Star(MainWindow.Options.ModelStarPath, "data_model_group_" + GroupNumber);
float[] SigmaValues = SigmaTable.GetColumn("rlnSigma2Noise").Select(v => float.Parse(v)).ToArray();
float[] Sigma2NoiseData = Sigma2Noise.GetHost(Intent.Write)[0];
Helper.ForEachElementFT(DimsCropped, (x, y, xx, yy, r, angle) =>
{
int ir = (int)r;
float val = 0;
if (ir < SigmaValues.Length && ir >= size / (50f / PixelSize) && ir < DimsCropped.X / 2)
{
if (SigmaValues[ir] != 0f)
val = 1f / SigmaValues[ir];
}
Sigma2NoiseData[y * (DimsCropped.X / 2 + 1) + x] = val;
});
float MaxSigma = MathHelper.Max(Sigma2NoiseData);
for (int i = 0; i < Sigma2NoiseData.Length; i++)
Sigma2NoiseData[i] /= MaxSigma;
Sigma2Noise.RemapToFT();
}
//Sigma2Noise.WriteMRC("d_sigma2noise.mrc");
#endregion
#region Initialize particle angles for both halves
float3[] ParticleAngles1 = new float3[NParticles1];
float3[] ParticleAngles2 = new float3[NParticles2];
for (int p = 0; p < NParticles1; p++)
ParticleAngles1[p] = new float3(float.Parse(tableIn.GetRowValue(RowIndices1[p], "rlnAngleRot")),
float.Parse(tableIn.GetRowValue(RowIndices1[p], "rlnAngleTilt")),
float.Parse(tableIn.GetRowValue(RowIndices1[p], "rlnAnglePsi")));
for (int p = 0; p < NParticles2; p++)
ParticleAngles2[p] = new float3(float.Parse(tableIn.GetRowValue(RowIndices2[p], "rlnAngleRot")),
float.Parse(tableIn.GetRowValue(RowIndices2[p], "rlnAngleTilt")),
float.Parse(tableIn.GetRowValue(RowIndices2[p], "rlnAnglePsi")));
#endregion
#region Prepare masks
Image Masks1, Masks2;
{
// Half 1
{
Image Volume = StageDataLoad.LoadMap(MainWindow.Options.MaskPath, new int2(1, 1), 0, typeof (float));
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
Image VolMaskFT = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
Image MasksFT = VolMaskFT.AsProjections(ParticleAngles1.Select(v => new float3(v.X * Helper.ToRad, v.Y * Helper.ToRad, v.Z * Helper.ToRad)).ToArray(),
new int2(DimsRegion),
MainWindow.Options.ProjectionOversample);
VolMaskFT.Dispose();
Masks1 = MasksFT.AsIFFT();
MasksFT.Dispose();
Masks1.RemapFromFT();
Parallel.ForEach(Masks1.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(50f, slice[i])) - 2) / 48f;
});
}
// Half 2
{
Image Volume = StageDataLoad.LoadMap(MainWindow.Options.MaskPath, new int2(1, 1), 0, typeof(float));
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
Image VolMaskFT = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
Image MasksFT = VolMaskFT.AsProjections(ParticleAngles2.Select(v => new float3(v.X * Helper.ToRad, v.Y * Helper.ToRad, v.Z * Helper.ToRad)).ToArray(),
new int2(DimsRegion),
MainWindow.Options.ProjectionOversample);
VolMaskFT.Dispose();
Masks2 = MasksFT.AsIFFT();
MasksFT.Dispose();
Masks2.RemapFromFT();
Parallel.ForEach(Masks2.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(50f, slice[i])) - 2) / 48f;
});
}
}
//Masks1.WriteMRC("d_masks1.mrc");
//Masks2.WriteMRC("d_masks2.mrc");
#endregion
#region Load and prepare references for both halves
Image VolRefFT1;
{
Image Volume = StageDataLoad.LoadMap(MainWindow.Options.ReferencePath, new int2(1, 1), 0, typeof(float));
//GPU.Normalize(Volume.GetDevice(Intent.Read), Volume.GetDevice(Intent.Write), (uint)Volume.ElementsReal, 1);
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
VolRefFT1 = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
}
VolRefFT1.FreeDevice();
Image VolRefFT2;
{
// Can't assume there is a second half, but certainly hope so
string Half2Path = MainWindow.Options.ReferencePath;
if (Half2Path.Contains("half1"))
Half2Path = Half2Path.Replace("half1", "half2");
Image Volume = StageDataLoad.LoadMap(Half2Path, new int2(1, 1), 0, typeof(float));
//GPU.Normalize(Volume.GetDevice(Intent.Read), Volume.GetDevice(Intent.Write), (uint)Volume.ElementsReal, 1);
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
VolRefFT2 = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
}
VolRefFT2.FreeDevice();
#endregion
#region Prepare particles: group and resize to DimsCropped
Image ParticleStackFT1 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles1 * Dims.Z / 3), true, true);
{
GPU.CreatePolishing(ParticleStack1.GetDevice(Intent.Read),
ParticleStackFT1.GetDevice(Intent.Write),
Masks1.GetDevice(Intent.Read),
new int2(DimsRegion),
DimsCropped,
NParticles1,
Dims.Z);
ParticleStack1.FreeDevice();
Masks1.Dispose();
/*Image Amps = ParticleStackFT1.AsIFFT();
Amps.RemapFromFT();
Amps.WriteMRC("d_particlestackft1.mrc");
Amps.Dispose();*/
}
Image ParticleStackFT2 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles2 * Dims.Z / 3), true, true);
{
GPU.CreatePolishing(ParticleStack2.GetDevice(Intent.Read),
ParticleStackFT2.GetDevice(Intent.Write),
Masks2.GetDevice(Intent.Read),
new int2(DimsRegion),
DimsCropped,
NParticles2,
Dims.Z);
ParticleStack1.FreeDevice();
Masks2.Dispose();
/*Image Amps = ParticleStackFT2.AsIFFT();
Amps.RemapFromFT();
Amps.WriteMRC("d_particlestackft2.mrc");
Amps.Dispose();*/
}
#endregion
Image Projections1 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles1 * Dims.Z / 3), true, true);
Image Projections2 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles2 * Dims.Z / 3), true, true);
Image Shifts1 = new Image(new int3(NParticles1, Dims.Z / 3, 1), false, true);
float3[] Angles1 = new float3[NParticles1 * Dims.Z / 3];
CTFStruct[] CTFParams1 = new CTFStruct[NParticles1 * Dims.Z / 3];
Image Shifts2 = new Image(new int3(NParticles2, Dims.Z / 3, 1), false, true);
float3[] Angles2 = new float3[NParticles2 * Dims.Z / 3];
CTFStruct[] CTFParams2 = new CTFStruct[NParticles2 * Dims.Z / 3];
float[] BFacs =
{
-3.86f,
0.00f,
-17.60f,
-35.24f,
-57.48f,
-93.51f,
-139.57f,
-139.16f
};
#region Initialize defocus and phase shift values
float[] InitialDefoci1 = new float[NParticles1 * (Dims.Z / 3)];
float[] InitialPhaseShifts1 = new float[NParticles1 * (Dims.Z / 3)];
float[] InitialDefoci2 = new float[NParticles2 * (Dims.Z / 3)];
float[] InitialPhaseShifts2 = new float[NParticles2 * (Dims.Z / 3)];
for (int z = 0, i = 0; z < Dims.Z / 3; z++)
{
for (int p = 0; p < NParticles1; p++, i++)
{
InitialDefoci1[i] = GridCTF.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
InitialPhaseShifts1[i] = GridCTFPhase.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
CTF Alt = CTF.GetCopy();
Alt.PixelSize = (decimal)PixelSize;
Alt.PixelSizeDelta = 0;
Alt.Defocus = (decimal)InitialDefoci1[i];
Alt.PhaseShift = (decimal)InitialPhaseShifts1[i];
//Alt.Bfactor = (decimal)BFacs[z];
CTFParams1[i] = Alt.ToStruct();
}
}
for (int z = 0, i = 0; z < Dims.Z / 3; z++)
{
for (int p = 0; p < NParticles2; p++, i++)
{
InitialDefoci2[i] = GridCTF.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
InitialPhaseShifts2[i] = GridCTFPhase.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
CTF Alt = CTF.GetCopy();
Alt.PixelSize = (decimal)PixelSize;
Alt.PixelSizeDelta = 0;
Alt.Defocus = (decimal)InitialDefoci2[i];
Alt.PhaseShift = (decimal)InitialPhaseShifts2[i];
//Alt.Bfactor = (decimal)BFacs[z];
CTFParams2[i] = Alt.ToStruct();
}
}
#endregion
#region SetPositions lambda
Action<double[]> SetPositions = input =>
{
float BorderZ = 0.5f / (Dims.Z / 3);
GridX = new CubicGrid(new int3(NParticles, 1, 2), input.Take(NParticles * 2).Select(v => (float)v).ToArray());
GridY = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 1).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredX = GridX.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] AlteredY = GridY.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
GridRot = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 2).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridTilt = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 3).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridPsi = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 4).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredRot = GridRot.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] AlteredTilt = GridTilt.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] AlteredPsi = GridPsi.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] ShiftData1 = Shifts1.GetHost(Intent.Write)[0];
float[] ShiftData2 = Shifts2.GetHost(Intent.Write)[0];
for (int z = 0; z < Dims.Z / 3; z++)
{
// Half 1
for (int p = 0; p < NParticles1; p++)
{
int i1 = z * NParticles1 + p;
int i = z * NParticles + p;
ShiftData1[i1 * 2] = AlteredX[i];
ShiftData1[i1 * 2 + 1] = AlteredY[i];
Angles1[i1] = new float3(AlteredRot[i] * 1f * Helper.ToRad, AlteredTilt[i] * 1f * Helper.ToRad, AlteredPsi[i] * 1f * Helper.ToRad);
}
// Half 2
for (int p = 0; p < NParticles2; p++)
{
int i2 = z * NParticles2 + p;
int i = z * NParticles + NParticles1 + p;
ShiftData2[i2 * 2] = AlteredX[i];
ShiftData2[i2 * 2 + 1] = AlteredY[i];
Angles2[i2] = new float3(AlteredRot[i] * 1f * Helper.ToRad, AlteredTilt[i] * 1f * Helper.ToRad, AlteredPsi[i] * 1f * Helper.ToRad);
}
}
};
#endregion
#region EvalIndividuals lambda
Func<double[], bool, double[]> EvalIndividuals = (input, redoProj) =>
{
SetPositions(input);
if (redoProj)
{
GPU.ProjectForward(VolRefFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Write),
VolRefFT1.Dims,
DimsCropped,
Helper.ToInterleaved(Angles1),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles1 * Dims.Z / 3));
GPU.ProjectForward(VolRefFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Write),
VolRefFT2.Dims,
DimsCropped,
Helper.ToInterleaved(Angles2),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles2 * Dims.Z / 3));
}
/*{
Image ProjectionsAmps = Projections1.AsIFFT();
ProjectionsAmps.RemapFromFT();
ProjectionsAmps.WriteMRC("d_projectionsamps1.mrc");
ProjectionsAmps.Dispose();
}
{
Image ProjectionsAmps = Projections2.AsIFFT();
ProjectionsAmps.RemapFromFT();
ProjectionsAmps.WriteMRC("d_projectionsamps2.mrc");
ProjectionsAmps.Dispose();
}*/
float[] Diff1 = new float[NParticles1];
float[] DiffAll1 = new float[NParticles1 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams1,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts1.GetDevice(Intent.Read),
Diff1,
DiffAll1,
(uint)NParticles1,
(uint)Dims.Z / 3);
float[] Diff2 = new float[NParticles2];
float[] DiffAll2 = new float[NParticles2 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams2,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts2.GetDevice(Intent.Read),
Diff2,
DiffAll2,
(uint)NParticles2,
(uint)Dims.Z / 3);
double[] DiffBoth = new double[NParticles];
for (int p = 0; p < NParticles1; p++)
DiffBoth[p] = Diff1[p];
for (int p = 0; p < NParticles2; p++)
DiffBoth[NParticles1 + p] = Diff2[p];
return DiffBoth;
};
#endregion
Func<double[], double> Eval = input =>
{
float Result = MathHelper.Mean(EvalIndividuals(input, true).Select(v => (float)v)) * NParticles;
Debug.WriteLine(Result);
return Result;
};
Func<double[], double[]> Grad = input =>
{
SetPositions(input);
GPU.ProjectForward(VolRefFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Write),
VolRefFT1.Dims,
DimsCropped,
Helper.ToInterleaved(Angles1),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles1 * Dims.Z / 3));
GPU.ProjectForward(VolRefFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Write),
VolRefFT2.Dims,
DimsCropped,
Helper.ToInterleaved(Angles2),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles2 * Dims.Z / 3));
double[] Result = new double[input.Length];
double Step = 0.1;
int NVariables = 10; // (Shift + Euler) * 2
for (int v = 0; v < NVariables; v++)
{
double[] InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
int iv = i / NParticles;
if (iv == v)
InputPlus[i] = input[i] + Step;
else
InputPlus[i] = input[i];
}
double[] ScorePlus = EvalIndividuals(InputPlus, v >= 4);
double[] InputMinus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
int iv = i / NParticles;
if (iv == v)
InputMinus[i] = input[i] - Step;
else
InputMinus[i] = input[i];
}
double[] ScoreMinus = EvalIndividuals(InputMinus, v >= 4);
for (int i = 0; i < NParticles; i++)
Result[v * NParticles + i] = (ScorePlus[i] - ScoreMinus[i]) / (Step * 2.0);
}
return Result;
};
double[] StartParams = new double[NParticles * 2 * 5];
for (int i = 0; i < NParticles * 2; i++)
{
int p = i % NParticles;
StartParams[NParticles * 2 * 0 + i] = 0;
StartParams[NParticles * 2 * 1 + i] = 0;
if (p < NParticles1)
{
StartParams[NParticles * 2 * 2 + i] = ParticleAngles1[p].X / 1.0;
StartParams[NParticles * 2 * 3 + i] = ParticleAngles1[p].Y / 1.0;
StartParams[NParticles * 2 * 4 + i] = ParticleAngles1[p].Z / 1.0;
}
else
{
p -= NParticles1;
StartParams[NParticles * 2 * 2 + i] = ParticleAngles2[p].X / 1.0;
StartParams[NParticles * 2 * 3 + i] = ParticleAngles2[p].Y / 1.0;
StartParams[NParticles * 2 * 4 + i] = ParticleAngles2[p].Z / 1.0;
}
}
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
Optimizer.Epsilon = 3e-7;
Optimizer.Maximize(StartParams);
#region Calculate particle quality for high frequencies
float[] ParticleQuality = new float[NParticles * (Dims.Z / 3)];
{
Sigma2Noise.Dispose();
Sigma2Noise = new Image(new int3(DimsCropped), true);
{
int GroupNumber = int.Parse(tableIn.GetRowValue(RowIndices[0], "rlnGroupNumber"));
//Star SigmaTable = new Star("D:\\rado27\\Refine3D\\run1_ct5_it009_half1_model.star", "data_model_group_" + GroupNumber);
Star SigmaTable = new Star(MainWindow.Options.ModelStarPath, "data_model_group_" + GroupNumber);
float[] SigmaValues = SigmaTable.GetColumn("rlnSigma2Noise").Select(v => float.Parse(v)).ToArray();
float[] Sigma2NoiseData = Sigma2Noise.GetHost(Intent.Write)[0];
Helper.ForEachElementFT(DimsCropped, (x, y, xx, yy, r, angle) =>
{
int ir = (int)r;
float val = 0;
if (ir < SigmaValues.Length && ir >= size / (4.0f / PixelSize) && ir < DimsCropped.X / 2)
{
if (SigmaValues[ir] != 0f)
val = 1f / SigmaValues[ir] / (ir * 3.14f);
}
Sigma2NoiseData[y * (DimsCropped.X / 2 + 1) + x] = val;
});
float MaxSigma = MathHelper.Max(Sigma2NoiseData);
for (int i = 0; i < Sigma2NoiseData.Length; i++)
Sigma2NoiseData[i] /= MaxSigma;
Sigma2Noise.RemapToFT();
}
//Sigma2Noise.WriteMRC("d_sigma2noiseScore.mrc");
SetPositions(StartParams);
GPU.ProjectForward(VolRefFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Write),
VolRefFT1.Dims,
DimsCropped,
Helper.ToInterleaved(Angles1),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles1 * Dims.Z / 3));
GPU.ProjectForward(VolRefFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Write),
VolRefFT2.Dims,
DimsCropped,
Helper.ToInterleaved(Angles2),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles2 * Dims.Z / 3));
float[] Diff1 = new float[NParticles1];
float[] ParticleQuality1 = new float[NParticles1 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams1,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts1.GetDevice(Intent.Read),
Diff1,
ParticleQuality1,
(uint)NParticles1,
(uint)Dims.Z / 3);
float[] Diff2 = new float[NParticles2];
float[] ParticleQuality2 = new float[NParticles2 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams2,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts2.GetDevice(Intent.Read),
Diff2,
ParticleQuality2,
(uint)NParticles2,
(uint)Dims.Z / 3);
for (int z = 0; z < Dims.Z / 3; z++)
{
for (int p = 0; p < NParticles1; p++)
ParticleQuality[z * NParticles + p] = ParticleQuality1[z * NParticles1 + p];
for (int p = 0; p < NParticles2; p++)
ParticleQuality[z * NParticles + NParticles1 + p] = ParticleQuality2[z * NParticles2 + p];
}
}
#endregion
lock (tableOut) // Only changing cell values, but better be safe in case table implementation changes later
{
GridX = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Take(NParticles * 2).Select(v => (float)v).ToArray());
GridY = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 1).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredX = GridX.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
float[] AlteredY = GridY.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
GridRot = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 2).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridTilt = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 3).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridPsi = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 4).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredRot = GridRot.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
float[] AlteredTilt = GridTilt.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
float[] AlteredPsi = GridPsi.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
for (int i = 0; i < TableOutIndices.Count; i++)
{
int p = i % NParticles;
int z = i / NParticles;
float Defocus = 0, PhaseShift = 0;
if (p < NParticles1)
{
Defocus = GridCTF.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
PhaseShift = GridCTFPhase.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
}
else
{
p -= NParticles1;
Defocus = GridCTF.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
PhaseShift = GridCTFPhase.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
}
tableOut.SetRowValue(TableOutIndices[i], "rlnOriginX", AlteredX[i].ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnOriginY", AlteredY[i].ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnAngleRot", (-AlteredRot[i]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnAngleTilt", (-AlteredTilt[i]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnAnglePsi", (-AlteredPsi[i]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnDefocusU", ((Defocus + (float)CTF.DefocusDelta / 2f) * 1e4f).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnDefocusV", ((Defocus - (float)CTF.DefocusDelta / 2f) * 1e4f).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnPhaseShift", (PhaseShift * 180f).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnCtfFigureOfMerit", (ParticleQuality[(z / 3) * NParticles + (i % NParticles)]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnMagnification", ((float)MainWindow.Options.CTFDetectorPixel * 10000f / PixelSize).ToString());
}
}
VolRefFT1.Dispose();
VolRefFT2.Dispose();
Projections1.Dispose();
Projections2.Dispose();
Sigma2Noise.Dispose();
ParticleStackFT1.Dispose();
ParticleStackFT2.Dispose();
Shifts1.Dispose();
Shifts2.Dispose();
CTFCoords.Dispose();
ShiftFactors.Dispose();
ParticleStack1.Dispose();
ParticleStack2.Dispose();
PSStack1.Dispose();
PSStack2.Dispose();
}
// Write movies to disk asynchronously, so the next micrograph can load.
Thread SaveThread = new Thread(() =>
{
GPU.SetDevice(CurrentDevice); // It's a separate thread, make sure it's using the same device
ParticleStackAll.WriteMRC(ParticleMoviesPath, ParticlesHeader);
//ParticleStackAll.WriteMRC("D:\\gala\\particlemovies\\" + RootName + "_particles.mrcs", ParticlesHeader);
ParticleStackAll.Dispose();
PSStackAll.WriteMRC(ParticleCTFMoviesPath);
//PSStackAll.WriteMRC("D:\\rado27\\particlectfmovies\\" + RootName + "_particlectf.mrcs");
PSStackAll.Dispose();
});
SaveThread.Start();
}
public Image AlignOneTiltMovie(Image tiltMovie, Image template, float initialAngle, float2 initialShift, float resolution)
{
float DownscaleFactor = (float)CTF.PixelSize * 2 / resolution;
//template.Bandpass(0.02f, DownscaleFactor, false);
//tiltMovie.Bandpass(0.02f, DownscaleFactor, false);
//template = template.AsPadded(new int2(template.Dims) - 512);
int2 DimsTemplate = new int2(template.Dims);
int2 DimsTemplateCoarse = new int2(DimsTemplate) * DownscaleFactor / 2 * 2;
int2 DimsFrame = new int2(tiltMovie.Dims);
int NFrames = tiltMovie.Dims.Z;
//GPU.Normalize(tiltMovie.GetDevice(Intent.Read),
// tiltMovie.GetDevice(Intent.Write),
// (uint)tiltMovie.ElementsSliceReal,
// (uint)tiltMovie.Dims.Z);
Image TemplateCoarse = template.AsScaled(DimsTemplateCoarse);
float GlobalAngle = initialAngle;
float ConditioningAngle = 180f / DimsFrame.X;
CubicGrid GridFrameX = new CubicGrid(new int3(1, 1, 1), initialShift.X, initialShift.X, Dimension.X);
CubicGrid GridFrameY = new CubicGrid(new int3(1, 1, 1), initialShift.Y, initialShift.Y, Dimension.X);
Action<double[]> SetFromVector = input =>
{
GlobalAngle = (float)input[0] * ConditioningAngle;
GridFrameX = new CubicGrid(GridFrameX.Dimensions, input.Skip(1).Take((int)GridFrameX.Dimensions.Elements()).Select(v => (float)v).ToArray());
GridFrameY = new CubicGrid(GridFrameY.Dimensions, input.Skip(1 + (int)GridFrameX.Dimensions.Elements()).Take((int)GridFrameY.Dimensions.Elements()).Select(v => (float)v).ToArray());
};
Func<double[], double[]> EvalIndividual = input =>
{
SetFromVector(input);
Image Transformed;
float GridStep = 1f / Math.Max(NFrames - 1, 1);
float2[] FrameShifts = new float2[NFrames];
for (int i = 0; i < NFrames; i++)
FrameShifts[i] = new float2(GridFrameX.GetInterpolated(new float3(0.5f, 0.5f, i * GridStep)),
GridFrameY.GetInterpolated(new float3(0.5f, 0.5f, i * GridStep)));
float[] FrameAngles = new float[NFrames].Select(v => -GlobalAngle * Helper.ToRad).ToArray();
Image MovieCopy = new Image(IntPtr.Zero, tiltMovie.Dims);
//MovieCopy.ShiftSlicesMassive(FrameShifts);
GPU.ShiftAndRotate2D(tiltMovie.GetDevice(Intent.Read),
MovieCopy.GetDevice(Intent.Write),
DimsFrame,
Helper.ToInterleaved(FrameShifts),
FrameAngles,
(uint)NFrames);
Transformed = MovieCopy.AsPadded(DimsTemplate);
MovieCopy.Dispose();
Transformed.MultiplySlices(template);
Image Sums = new Image(IntPtr.Zero, new int3(NFrames, 1, 1));
GPU.Sum(Transformed.GetDevice(Intent.Read),
Sums.GetDevice(Intent.Write),
(uint)Transformed.ElementsSliceReal,
(uint)NFrames);
Transformed.Dispose();
double[] Result = new double[NFrames];
for (int i = 0; i < NFrames; i++)
Result[i] = Sums.GetHost(Intent.Read)[0][i] / Transformed.ElementsSliceReal * 100;
return Result;
};
Func<double[], double> Eval = input =>
{
double[] Scores = EvalIndividual(input);
double Score = Scores.Sum();
Debug.WriteLine(Score);
return Score;
};
Func<double[], double[]> Grad = input =>
{
double Delta = 0.1 / DownscaleFactor;
double[] Result = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
double[] InputPlus = input.ToArray();
InputPlus[i] += Delta;
double ScorePlus = EvalIndividual(InputPlus).Sum();
double[] InputMinus = input.ToArray();
InputMinus[i] -= Delta;
double ScoreMinus = EvalIndividual(InputMinus).Sum();
Result[i] = (ScorePlus - ScoreMinus) / (Delta * 2);
}
return Result;
};
List<double> StartList = new List<double>();
StartList.Add(GlobalAngle / ConditioningAngle);
StartList.AddRange(GridFrameX.FlatValues.Select(v => (double)v));
StartList.AddRange(GridFrameY.FlatValues.Select(v => (double)v));
double[] StartVector = StartList.ToArray();
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartVector.Length, Eval, Grad);
Optimizer.Maximize(StartVector);
TemplateCoarse.Dispose();
return null;
}
public void ProcessCTFOneAngle(Image angleImage,
float angle,
bool fromScratch,
bool fixAstigmatism,
float2 astigmatism,
CTF previousCTF,
CubicGrid previousGrid,
Cubic1D previousBackground,
Cubic1D previousScale,
out CTF thisCTF,
out CubicGrid thisGrid,
out float2[] thisPS1D,
out Cubic1D thisBackground,
out Cubic1D thisScale,
out Image thisPS2D)
{
CTF TempCTF = previousCTF != null ? previousCTF.GetCopy() : new CTF();
float2[] TempPS1D = null;
Cubic1D TempBackground = null, TempScale = null;
CubicGrid TempGrid = null;
#region Dimensions and grids
int NFrames = angleImage.Dims.Z;
int2 DimsImage = angleImage.DimsSlice;
int2 DimsRegion = new int2(MainWindow.Options.CTFWindow, MainWindow.Options.CTFWindow);
float OverlapFraction = 0.5f;
int2 DimsPositionGrid;
int3[] PositionGrid = Helper.GetEqualGridSpacing(DimsImage, new int2(DimsRegion.X, DimsRegion.Y), OverlapFraction, out DimsPositionGrid);
int NPositions = (int)DimsPositionGrid.Elements();
if (previousGrid == null)
TempGrid = new CubicGrid(new int3(2, 2, 1));
else
TempGrid = new CubicGrid(new int3(2, 2, 1), previousGrid.FlatValues);
bool CTFSpace = true;
bool CTFTime = false;
int3 CTFSpectraGrid = new int3(DimsPositionGrid.X, DimsPositionGrid.Y, 1);
int MinFreqInclusive = (int)(MainWindow.Options.CTFRangeMin * DimsRegion.X / 2);
int MaxFreqExclusive = (int)(MainWindow.Options.CTFRangeMax * DimsRegion.X / 2);
int NFreq = MaxFreqExclusive - MinFreqInclusive;
#endregion
#region Allocate GPU memory
Image CTFSpectra = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.X, (int)CTFSpectraGrid.Elements()), true);
Image CTFMean = new Image(IntPtr.Zero, new int3(DimsRegion), true);
Image CTFCoordsCart = new Image(new int3(DimsRegion), true, true);
Image CTFCoordsPolarTrimmed = new Image(new int3(NFreq, DimsRegion.X, 1), false, true);
#endregion
// Extract movie regions, create individual spectra in Cartesian coordinates and their mean.
#region Create spectra
GPU.CreateSpectra(angleImage.GetDevice(Intent.Read),
DimsImage,
NFrames,
PositionGrid,
NPositions,
DimsRegion,
CTFSpectraGrid,
CTFSpectra.GetDevice(Intent.Write),
CTFMean.GetDevice(Intent.Write));
angleImage.FreeDevice(); // Won't need it in this method anymore.
#endregion
// Populate address arrays for later.
#region Init addresses
{
float2[] CoordsData = new float2[CTFCoordsCart.ElementsSliceComplex];
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) => CoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r, a));
CTFCoordsCart.UpdateHostWithComplex(new[] { CoordsData });
CoordsData = new float2[NFreq * DimsRegion.X];
Helper.ForEachElement(CTFCoordsPolarTrimmed.DimsSlice, (x, y) =>
{
float Angle = ((float)y / DimsRegion.X + 0.5f) * (float)Math.PI;
float Ny = 1f / DimsRegion.X;
CoordsData[y * NFreq + x] = new float2((x + MinFreqInclusive) * Ny, Angle);
});
CTFCoordsPolarTrimmed.UpdateHostWithComplex(new[] { CoordsData });
}
#endregion
// Retrieve average 1D spectrum from CTFMean (not corrected for astigmatism yet).
#region Initial 1D spectrum
{
Image CTFAverage1D = new Image(IntPtr.Zero, new int3(DimsRegion.X / 2, 1, 1));
GPU.CTFMakeAverage(CTFMean.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
(uint)CTFMean.ElementsSliceReal,
(uint)DimsRegion.X,
new[] { new CTF().ToStruct() },
new CTF().ToStruct(),
0,
(uint)DimsRegion.X / 2,
null,
1,
CTFAverage1D.GetDevice(Intent.Write));
//CTFAverage1D.WriteMRC("CTFAverage1D.mrc");
float[] CTFAverage1DData = CTFAverage1D.GetHost(Intent.Read)[0];
float2[] ForPS1D = new float2[DimsRegion.X / 2];
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, (float)Math.Round(CTFAverage1DData[i], 4));
TempPS1D = ForPS1D;
CTFAverage1D.Dispose();
}
#endregion
#region Background fitting methods
Action UpdateBackgroundFit = () =>
{
float2[] ForPS1D = TempPS1D.Skip(Math.Max(5, MinFreqInclusive / 2)).ToArray();
Cubic1D.FitCTF(ForPS1D,
v => v.Select(x => TempCTF.Get1D(x / (float)TempCTF.PixelSize, true)).ToArray(),
TempCTF.GetZeros(),
TempCTF.GetPeaks(),
out TempBackground,
out TempScale);
};
Action<bool> UpdateRotationalAverage = keepbackground =>
{
float[] MeanData = CTFMean.GetHost(Intent.Read)[0];
Image CTFMeanCorrected = new Image(new int3(DimsRegion), true);
float[] MeanCorrectedData = CTFMeanCorrected.GetHost(Intent.Write)[0];
// Subtract current background estimate from spectra, populate coords.
Helper.ForEachElementFT(DimsRegion,
(x, y, xx, yy, r, a) =>
{
int i = y * (DimsRegion.X / 2 + 1) + x;
MeanCorrectedData[i] = MeanData[i] - TempBackground.Interp(r / DimsRegion.X);
});
Image CTFAverage1D = new Image(IntPtr.Zero, new int3(DimsRegion.X / 2, 1, 1));
GPU.CTFMakeAverage(CTFMeanCorrected.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
(uint)CTFMeanCorrected.DimsEffective.ElementsSlice(),
(uint)DimsRegion.X,
new[] { TempCTF.ToStruct() },
TempCTF.ToStruct(),
0,
(uint)DimsRegion.X / 2,
null,
1,
CTFAverage1D.GetDevice(Intent.Write));
//CTFAverage1D.WriteMRC("CTFAverage1D.mrc");
float[] RotationalAverageData = CTFAverage1D.GetHost(Intent.Read)[0];
float2[] ForPS1D = new float2[TempPS1D.Length];
if (keepbackground)
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, RotationalAverageData[i] + TempBackground.Interp((float)i / DimsRegion.X));
else
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, RotationalAverageData[i]);
MathHelper.UnNaN(ForPS1D);
TempPS1D = ForPS1D;
CTFMeanCorrected.Dispose();
CTFAverage1D.Dispose();
};
#endregion
// Fit background to currently best average (not corrected for astigmatism yet).
{
float2[] ForPS1D = TempPS1D.Skip(MinFreqInclusive).Take(Math.Max(2, NFreq / 2)).ToArray();
float[] CurrentBackground;
//if (previousBackground == null)
{
int NumNodes = Math.Max(3, (int)((MainWindow.Options.CTFRangeMax - MainWindow.Options.CTFRangeMin) * 5M));
TempBackground = Cubic1D.Fit(ForPS1D, NumNodes); // This won't fit falloff and scale, because approx function is 0
CurrentBackground = TempBackground.Interp(TempPS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq / 2).ToArray();
}
/*else
{
CurrentBackground = previousBackground.Interp(TempPS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq / 2).ToArray();
TempBackground = new Cubic1D(previousBackground.Data);
}*/
float[] Subtracted1D = new float[ForPS1D.Length];
for (int i = 0; i < ForPS1D.Length; i++)
Subtracted1D[i] = ForPS1D[i].Y - CurrentBackground[i];
MathHelper.NormalizeInPlace(Subtracted1D);
float ZMin = (float)MainWindow.Options.CTFZMin;
float ZMax = (float)MainWindow.Options.CTFZMax;
float PhaseMin = 0f;
float PhaseMax = MainWindow.Options.CTFDoPhase ? 1f : 0f;
if (previousCTF != null)
{
ZMin = (float)previousCTF.Defocus - 0.5f;
ZMax = (float)previousCTF.Defocus + 0.5f;
if (PhaseMax > 0)
{
PhaseMin = (float)previousCTF.PhaseShift - 0.3f;
PhaseMax = (float)previousCTF.PhaseShift + 0.3f;
}
}
float ZStep = (ZMax - ZMin) / 100f;
float BestZ = 0, BestPhase = 0, BestScore = -999;
for (float z = ZMin; z <= ZMax + 1e-5f; z += ZStep)
{
for (float p = PhaseMin; p <= PhaseMax; p += 0.01f)
{
CTF CurrentParams = new CTF
{
PixelSize = (MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5M,
Defocus = (decimal)z,
PhaseShift = (decimal)p,
Cs = MainWindow.Options.CTFCs,
Voltage = MainWindow.Options.CTFVoltage,
Amplitude = MainWindow.Options.CTFAmplitude
};
float[] SimulatedCTF = CurrentParams.Get1D(TempPS1D.Length, true).Skip(MinFreqInclusive).Take(Math.Max(2, NFreq / 2)).ToArray();
MathHelper.NormalizeInPlace(SimulatedCTF);
float Score = MathHelper.CrossCorrelate(Subtracted1D, SimulatedCTF);
if (Score > BestScore)
{
BestScore = Score;
BestZ = z;
BestPhase = p;
}
}
}
TempCTF = new CTF
{
PixelSize = (MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5M,
Defocus = (decimal)BestZ,
PhaseShift = (decimal)BestPhase,
Cs = MainWindow.Options.CTFCs,
Voltage = MainWindow.Options.CTFVoltage,
Amplitude = MainWindow.Options.CTFAmplitude
};
UpdateRotationalAverage(true); // This doesn't have a nice background yet.
UpdateBackgroundFit(); // Now get a reasonably nice background.
}
// Fit defocus, (phase shift), (astigmatism) to average background-subtracted spectrum,
// which is in polar coords at this point (for equal weighting of all frequencies).
#region Grid search
if (fromScratch)
{
Image CTFMeanPolarTrimmed = CTFMean.AsPolar((uint)MinFreqInclusive, (uint)(MinFreqInclusive + NFreq / 1));
// Subtract current background.
Image CurrentBackground = new Image(TempBackground.Interp(TempPS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq / 1).ToArray());
CTFMeanPolarTrimmed.SubtractFromLines(CurrentBackground);
CurrentBackground.Dispose();
// Normalize for CC (not strictly needed, but it's converted for fp16 later, so let's be on the safe side of the fp16 range.
GPU.Normalize(CTFMeanPolarTrimmed.GetDevice(Intent.Read), CTFMeanPolarTrimmed.GetDevice(Intent.Write), (uint)CTFMeanPolarTrimmed.ElementsReal, 1);
//CTFMeanPolarTrimmed.WriteMRC("ctfmeanpolartrimmed.mrc");
CTF StartParams = new CTF
{
PixelSize = (MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5M,
PixelSizeDelta = Math.Abs(MainWindow.Options.CTFPixelMax - MainWindow.Options.CTFPixelMin),
PixelSizeAngle = MainWindow.Options.CTFPixelAngle,
Defocus = TempCTF.Defocus, // (MainWindow.Options.CTFZMin + MainWindow.Options.CTFZMax) * 0.5M,
DefocusDelta = 0,
DefocusAngle = 0,
PhaseShift = TempCTF.PhaseShift,
Cs = MainWindow.Options.CTFCs,
Voltage = MainWindow.Options.CTFVoltage,
Amplitude = MainWindow.Options.CTFAmplitude
};
CTFFitStruct FitParams = new CTFFitStruct
{
Defocus = new float3(-0.4e-6f,
0.4e-6f,
0.025e-6f),
Defocusdelta = new float3(0, 0.8e-6f, 0.02e-6f),
Astigmatismangle = new float3(0, 2 * (float)Math.PI, 1 * (float)Math.PI / 18),
Phaseshift = MainWindow.Options.CTFDoPhase ? new float3(-0.2f * (float)Math.PI, 0.2f * (float)Math.PI, 0.025f * (float)Math.PI) : new float3(0, 0, 0)
};
CTFStruct ResultStruct = GPU.CTFFitMean(CTFMeanPolarTrimmed.GetDevice(Intent.Read),
CTFCoordsPolarTrimmed.GetDevice(Intent.Read),
CTFMeanPolarTrimmed.DimsSlice,
StartParams.ToStruct(),
FitParams,
true);
TempCTF.FromStruct(ResultStruct);
TempCTF.Defocus = Math.Max(TempCTF.Defocus, MainWindow.Options.CTFZMin);
CTFMeanPolarTrimmed.Dispose();
UpdateRotationalAverage(true); // This doesn't have a nice background yet.
UpdateBackgroundFit(); // Now get a reasonably nice background.
UpdateRotationalAverage(true); // This time, with the nice background.
UpdateBackgroundFit(); // Make the background even nicer!
}
else if (previousCTF != null)
{
TempCTF.DefocusDelta = previousCTF.DefocusDelta;
TempCTF.DefocusAngle = previousCTF.DefocusAngle;
}
if (fixAstigmatism)
{
TempCTF.DefocusDelta = (decimal)astigmatism.X;
TempCTF.DefocusAngle = (decimal)astigmatism.Y;
}
#endregion
if (previousGrid == null)
TempGrid = new CubicGrid(TempGrid.Dimensions, (float)TempCTF.Defocus, (float)TempCTF.Defocus, Dimension.X);
// Do BFGS optimization of defocus, astigmatism and phase shift,
// using 2D simulation for comparison
#region BFGS
bool[] CTFSpectraConsider = new bool[CTFSpectraGrid.Elements()];
for (int i = 0; i < CTFSpectraConsider.Length; i++)
CTFSpectraConsider[i] = true;
int NCTFSpectraConsider = CTFSpectraConsider.Length;
{
Image CTFSpectraPolarTrimmed = CTFSpectra.AsPolar((uint)MinFreqInclusive, (uint)(MinFreqInclusive + NFreq));
CTFSpectra.FreeDevice(); // This will only be needed again for the final PS1D.
#region Create background and scale
float[] CurrentScale = TempScale.Interp(TempPS1D.Select(p => p.X).ToArray());
Image CTFSpectraScale = new Image(new int3(NFreq, DimsRegion.X, 1));
float[] CTFSpectraScaleData = CTFSpectraScale.GetHost(Intent.Write)[0];
// Trim polar to relevant frequencies, and populate coordinates.
Parallel.For(0, DimsRegion.X, y =>
{
for (int x = 0; x < NFreq; x++)
CTFSpectraScaleData[y * NFreq + x] = CurrentScale[x + MinFreqInclusive];
});
//CTFSpectraScale.WriteMRC("ctfspectrascale.mrc");
// Background is just 1 line since we're in polar.
Image CurrentBackground = new Image(TempBackground.Interp(TempPS1D.Select(p => p.X).ToArray()).Skip(MinFreqInclusive).Take(NFreq).ToArray());
#endregion
CTFSpectraPolarTrimmed.SubtractFromLines(CurrentBackground);
CurrentBackground.Dispose();
// Normalize background-subtracted spectra.
GPU.Normalize(CTFSpectraPolarTrimmed.GetDevice(Intent.Read),
CTFSpectraPolarTrimmed.GetDevice(Intent.Write),
(uint)CTFSpectraPolarTrimmed.ElementsSliceReal,
(uint)CTFSpectraGrid.Elements());
//CTFSpectraPolarTrimmed.WriteMRC("ctfspectrapolartrimmed.mrc");
#region Convert to fp16
Image CTFSpectraPolarTrimmedHalf = CTFSpectraPolarTrimmed.AsHalf();
CTFSpectraPolarTrimmed.Dispose();
Image CTFSpectraScaleHalf = CTFSpectraScale.AsHalf();
CTFSpectraScale.Dispose();
Image CTFCoordsPolarTrimmedHalf = CTFCoordsPolarTrimmed.AsHalf();
#endregion
// Wiggle weights show how the defocus on the spectra grid is altered
// by changes in individual anchor points of the spline grid.
// They are used later to compute the dScore/dDefocus values for each spectrum
// only once, and derive the values for each anchor point from them.
float[][] WiggleWeights = TempGrid.GetWiggleWeights(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0));
// Helper method for getting CTFStructs for the entire spectra grid.
Func<double[], CTF, float[], CTFStruct[]> EvalGetCTF = (input, ctf, defocusValues) =>
{
decimal AlteredPhase = MainWindow.Options.CTFDoPhase ? (decimal)input[input.Length - 3] : 0;
decimal AlteredDelta = (decimal)input[input.Length - 2];
decimal AlteredAngle = (decimal)(input[input.Length - 1] * 20 / (Math.PI / 180));
CTF Local = ctf.GetCopy();
Local.PhaseShift = AlteredPhase;
Local.DefocusDelta = AlteredDelta;
Local.DefocusAngle = AlteredAngle;
CTFStruct LocalStruct = Local.ToStruct();
CTFStruct[] LocalParams = new CTFStruct[defocusValues.Length];
for (int i = 0; i < LocalParams.Length; i++)
{
LocalParams[i] = LocalStruct;
LocalParams[i].Defocus = defocusValues[i] * -1e-6f;
}
return LocalParams;
};
// Simulate with adjusted CTF, compare to originals
#region Eval and Gradient methods
Func<double[], double> Eval = input =>
{
CubicGrid Altered = new CubicGrid(TempGrid.Dimensions, input.Take((int)TempGrid.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] DefocusValues = Altered.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0));
CTFStruct[] LocalParams = EvalGetCTF(input, TempCTF, DefocusValues);
float[] Result = new float[LocalParams.Length];
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
Result,
(uint)LocalParams.Length);
float Score = 0;
for (int i = 0; i < Result.Length; i++)
if (CTFSpectraConsider[i])
Score += Result[i];
Score /= NCTFSpectraConsider;
if (float.IsNaN(Score) || float.IsInfinity(Score))
throw new Exception("Bad score.");
return (1.0 - Score) * 1000.0;
};
Func<double[], double[]> Gradient = input =>
{
const float Step = 0.005f;
double[] Result = new double[input.Length];
// In 0D grid case, just get gradient for all 4 parameters.
// In 1+D grid case, do simple gradient for astigmatism and phase...
int StartComponent = input.Length - 3;
//int StartComponent = 0;
for (int i = StartComponent; i < input.Length; i++)
{
if (fixAstigmatism && i > StartComponent)
continue;
double[] UpperInput = new double[input.Length];
input.CopyTo(UpperInput, 0);
UpperInput[i] += Step;
double UpperValue = Eval(UpperInput);
double[] LowerInput = new double[input.Length];
input.CopyTo(LowerInput, 0);
LowerInput[i] -= Step;
double LowerValue = Eval(LowerInput);
Result[i] = (UpperValue - LowerValue) / (2f * Step);
}
float[] ResultPlus = new float[CTFSpectraGrid.Elements()];
float[] ResultMinus = new float[CTFSpectraGrid.Elements()];
// ..., take shortcut for defoci...
{
{
CubicGrid AlteredPlus = new CubicGrid(TempGrid.Dimensions, input.Take((int)TempGrid.Dimensions.Elements()).Select(v => (float)v + Step).ToArray());
float[] DefocusValues = AlteredPlus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0));
CTFStruct[] LocalParams = EvalGetCTF(input, TempCTF, DefocusValues);
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
ResultPlus,
(uint)LocalParams.Length);
}
{
CubicGrid AlteredMinus = new CubicGrid(TempGrid.Dimensions, input.Take((int)TempGrid.Dimensions.Elements()).Select(v => (float)v - Step).ToArray());
float[] DefocusValues = AlteredMinus.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0));
CTFStruct[] LocalParams = EvalGetCTF(input, TempCTF, DefocusValues);
GPU.CTFCompareToSim(CTFSpectraPolarTrimmedHalf.GetDevice(Intent.Read),
CTFCoordsPolarTrimmedHalf.GetDevice(Intent.Read),
CTFSpectraScaleHalf.GetDevice(Intent.Read),
(uint)CTFSpectraPolarTrimmedHalf.ElementsSliceReal,
LocalParams,
ResultMinus,
(uint)LocalParams.Length);
}
float[] LocalGradients = new float[ResultPlus.Length];
for (int i = 0; i < LocalGradients.Length; i++)
LocalGradients[i] = ResultMinus[i] - ResultPlus[i];
// Now compute gradients per grid anchor point using the precomputed individual gradients and wiggle factors.
Parallel.For(0, TempGrid.Dimensions.Elements(), i => Result[i] = MathHelper.ReduceWeighted(LocalGradients, WiggleWeights[i]) / LocalGradients.Length / (2f * Step) * 1000f);
}
foreach (var i in Result)
if (double.IsNaN(i) || double.IsInfinity(i))
throw new Exception("Bad score.");
return Result;
};
#endregion
#region Minimize first time with potential outpiers
double[] StartParams = new double[TempGrid.Dimensions.Elements() + 3];
for (int i = 0; i < TempGrid.Dimensions.Elements(); i++)
StartParams[i] = TempGrid.FlatValues[i];
StartParams[StartParams.Length - 3] = (double)TempCTF.PhaseShift;
StartParams[StartParams.Length - 2] = (double)TempCTF.DefocusDelta;
StartParams[StartParams.Length - 1] = (double)TempCTF.DefocusAngle / 20 * (Math.PI / 180);
// Compute correlation for individual spectra, and throw away those that are >.75 sigma worse than mean.
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Gradient)
{
Past = 1,
Delta = 1e-6,
MaxLineSearch = 15,
Corrections = 20
};
Optimizer.Minimize(StartParams);
#endregion
#region Retrieve parameters
TempCTF.Defocus = (decimal)MathHelper.Mean(Optimizer.Solution.Take((int)TempGrid.Dimensions.Elements()).Select(v => (float)v));
TempCTF.PhaseShift = (decimal)Optimizer.Solution[StartParams.Length - 3];
TempCTF.DefocusDelta = (decimal)Optimizer.Solution[StartParams.Length - 2];
TempCTF.DefocusAngle = (decimal)(Optimizer.Solution[StartParams.Length - 1] * 20 / (Math.PI / 180));
if (TempCTF.DefocusDelta < 0)
{
TempCTF.DefocusAngle += 90;
TempCTF.DefocusDelta *= -1;
}
TempCTF.DefocusAngle = ((int)TempCTF.DefocusAngle + 180 * 99) % 180;
TempGrid = new CubicGrid(TempGrid.Dimensions, Optimizer.Solution.Take((int)TempGrid.Dimensions.Elements()).Select(v => (float)v).ToArray());
#endregion
// Dispose GPU resources manually because GC can't be bothered to do it in time.
CTFSpectraPolarTrimmedHalf.Dispose();
CTFCoordsPolarTrimmedHalf.Dispose();
CTFSpectraScaleHalf.Dispose();
#region Get nicer envelope fit
{
{
Image CTFSpectraBackground = new Image(new int3(DimsRegion), true);
float[] CTFSpectraBackgroundData = CTFSpectraBackground.GetHost(Intent.Write)[0];
// Construct background in Cartesian coordinates.
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) =>
{
CTFSpectraBackgroundData[y * CTFSpectraBackground.DimsEffective.X + x] = TempBackground.Interp(r / DimsRegion.X);
});
CTFSpectra.SubtractFromSlices(CTFSpectraBackground);
float[] DefocusValues = TempGrid.GetInterpolatedNative(CTFSpectraGrid, new float3(DimsRegion.X / 2f / DimsImage.X, DimsRegion.Y / 2f / DimsImage.Y, 0));
CTFStruct[] LocalParams = DefocusValues.Select(v =>
{
CTF Local = TempCTF.GetCopy();
Local.Defocus = (decimal)v + 0.0M;
return Local.ToStruct();
}).ToArray();
Image CTFAverage1D = new Image(IntPtr.Zero, new int3(DimsRegion.X / 2, 1, 1));
CTF CTFAug = TempCTF.GetCopy();
CTFAug.Defocus += 0.0M;
GPU.CTFMakeAverage(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceReal,
(uint)DimsRegion.X,
LocalParams,
CTFAug.ToStruct(),
0,
(uint)DimsRegion.X / 2,
CTFSpectraConsider.Select(v => v ? 1 : 0).ToArray(),
(uint)CTFSpectraGrid.Elements(),
CTFAverage1D.GetDevice(Intent.Write));
CTFSpectra.AddToSlices(CTFSpectraBackground);
float[] RotationalAverageData = CTFAverage1D.GetHost(Intent.Read)[0];
float2[] ForPS1D = new float2[TempPS1D.Length];
for (int i = 0; i < ForPS1D.Length; i++)
ForPS1D[i] = new float2((float)i / DimsRegion.X, (float)Math.Round(RotationalAverageData[i], 4) + TempBackground.Interp((float)i / DimsRegion.X));
MathHelper.UnNaN(ForPS1D);
TempPS1D = ForPS1D;
CTFSpectraBackground.Dispose();
CTFAverage1D.Dispose();
CTFSpectra.FreeDevice();
}
TempCTF.Defocus = Math.Max(TempCTF.Defocus, MainWindow.Options.CTFZMin);
UpdateBackgroundFit();
}
#endregion
}
#endregion
// Subtract background from 2D average and write it to disk.
// This image is used for quick visualization purposes only.
#region PS2D update
{
int3 DimsAverage = new int3(DimsRegion.X, DimsRegion.X / 2, 1);
float[] Average2DData = new float[DimsAverage.Elements()];
float[] OriginalAverageData = CTFMean.GetHost(Intent.Read)[0];
for (int y = 0; y < DimsAverage.Y; y++)
{
int yy = y * y;
for (int x = 0; x < DimsAverage.Y; x++)
{
int xx = DimsRegion.X / 2 - x - 1;
xx *= xx;
float r = (float)Math.Sqrt(xx + yy) / DimsRegion.X;
Average2DData[y * DimsAverage.X + x] = OriginalAverageData[(y + DimsRegion.X / 2) * (DimsRegion.X / 2 + 1) + x] - TempBackground.Interp(r);
}
for (int x = 0; x < DimsRegion.X / 2; x++)
{
int xx = x * x;
float r = (float)Math.Sqrt(xx + yy) / DimsRegion.X;
Average2DData[y * DimsAverage.X + x + DimsRegion.X / 2] = OriginalAverageData[(DimsRegion.X / 2 - y) * (DimsRegion.X / 2 + 1) + (DimsRegion.X / 2 - 1 - x)] - TempBackground.Interp(r);
}
}
thisPS2D = new Image(Average2DData, DimsAverage);
}
#endregion
for (int i = 0; i < TempPS1D.Length; i++)
TempPS1D[i].Y -= TempBackground.Interp(TempPS1D[i].X);
CTFSpectra.Dispose();
CTFMean.Dispose();
CTFCoordsCart.Dispose();
CTFCoordsPolarTrimmed.Dispose();
thisPS1D = TempPS1D;
thisBackground = TempBackground;
thisScale = TempScale;
thisCTF = TempCTF;
thisGrid = TempGrid;
}
public void PerformOptimizationStep(Star tableIn, Image tiltStack, int size, int3 volumeDimensions, Dictionary<int, Projector> references, float resolution, Dictionary<int, Projector> outReconstructions, Dictionary<int, Projector> outCTFReconstructions)
{
VolumeDimensions = volumeDimensions;
#region Get rows from table
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName + "."))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return;
int NParticles = RowIndices.Count;
#endregion
#region Make sure all columns and directories are there
if (!tableIn.HasColumn("rlnImageName"))
tableIn.AddColumn("rlnImageName");
if (!tableIn.HasColumn("rlnCtfImage"))
tableIn.AddColumn("rlnCtfImage");
if (!tableIn.HasColumn("rlnParticleSelectZScore"))
tableIn.AddColumn("rlnParticleSelectZScore");
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
#endregion
#region Get subtomo positions from table
float3[] ParticleOrigins = new float3[NParticles];
float3[] ParticleOrigins2 = new float3[NParticles];
float3[] ParticleAngles = new float3[NParticles];
float3[] ParticleAngles2 = new float3[NParticles];
int[] ParticleSubset = new int[NParticles];
{
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnPosZ = tableIn.GetColumn("rlnCoordinateZ");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
string[] ColumnOriginZ = tableIn.GetColumn("rlnOriginZ");
string[] ColumnAngleRot = tableIn.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = tableIn.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = tableIn.GetColumn("rlnAnglePsi");
string[] ColumnSubset = tableIn.GetColumn("rlnRandomSubset");
string[] ColumnPosX2 = tableIn.GetColumn("rlnOriginXPrior");
string[] ColumnPosY2 = tableIn.GetColumn("rlnOriginYPrior");
string[] ColumnPosZ2 = tableIn.GetColumn("rlnOriginZPrior");
string[] ColumnAngleRot2 = tableIn.GetColumn("rlnAngleRotPrior");
string[] ColumnAngleTilt2 = tableIn.GetColumn("rlnAngleTiltPrior");
string[] ColumnAnglePsi2 = tableIn.GetColumn("rlnAnglePsiPrior");
for (int i = 0; i < NParticles; i++)
{
float3 Pos = new float3(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosZ[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Pos2 = Pos;
//if (ColumnPosX2 != null && ColumnPosY2 != null && ColumnPosZ2 != null)
// Pos2 = new float3(float.Parse(ColumnPosX2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnPosY2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnPosZ2[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Shift = new float3(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginZ[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleOrigins[i] = Pos - Shift;
ParticleOrigins2[i] = Pos2 - Shift;
float3 Angle = new float3(float.Parse(ColumnAngleRot[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAngleTilt[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAnglePsi[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Angle2 = Angle;
//if (ColumnAngleRot2 != null && ColumnAngleTilt2 != null && ColumnAnglePsi2 != null)
// Angle2 = new float3(float.Parse(ColumnAngleRot2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnAngleTilt2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnAnglePsi2[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleAngles[i] = Angle;
ParticleAngles2[i] = Angle2;
ParticleSubset[i] = int.Parse(ColumnSubset[RowIndices[i]]);
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", ParticleOrigins[i].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", ParticleOrigins[i].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateZ", ParticleOrigins[i].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginZ", "0.0");
}
}
#endregion
#region Deal with subsets
List<int> SubsetIDs = new List<int>();
foreach (var i in ParticleSubset)
if (!SubsetIDs.Contains(i))
SubsetIDs.Add(i);
SubsetIDs.Sort();
// For each subset, create a list of its particle IDs
Dictionary<int, List<int>> SubsetParticleIDs = SubsetIDs.ToDictionary(subsetID => subsetID, subsetID => new List<int>());
for (int i = 0; i < ParticleSubset.Length; i++)
SubsetParticleIDs[ParticleSubset[i]].Add(i);
foreach (var list in SubsetParticleIDs.Values)
list.Sort();
// Note where each subset starts and ends in a unified, sorted (by subset) particle ID list
Dictionary<int, Tuple<int, int>> SubsetRanges = new Dictionary<int, Tuple<int, int>>();
{
int Start = 0;
foreach (var pair in SubsetParticleIDs)
{
SubsetRanges.Add(pair.Key, new Tuple<int, int>(Start, Start + pair.Value.Count));
Start += pair.Value.Count;
}
}
List<int> SubsetContinuousIDs = new List<int>();
foreach (var pair in SubsetParticleIDs)
SubsetContinuousIDs.AddRange(pair.Value);
// Reorder particle information to match the order of SubsetContinuousIDs
ParticleOrigins = SubsetContinuousIDs.Select(i => ParticleOrigins[i]).ToArray();
ParticleOrigins2 = SubsetContinuousIDs.Select(i => ParticleOrigins2[i]).ToArray();
ParticleAngles = SubsetContinuousIDs.Select(i => ParticleAngles[i]).ToArray();
ParticleAngles2 = SubsetContinuousIDs.Select(i => ParticleAngles2[i]).ToArray();
ParticleSubset = SubsetContinuousIDs.Select(i => ParticleSubset[i]).ToArray();
#endregion
if (GridMovementX.Dimensions.Elements() == 1)
{
int MaxSlice = SubsetRanges.Last().Value.Item2 > 100 ? 1 : 1;
GridMovementX = new CubicGrid(new int3(MaxSlice, MaxSlice, NTilts));
GridMovementY = new CubicGrid(new int3(MaxSlice, MaxSlice, NTilts));
//GridLocalX = new CubicGrid(new int3(4, 4, 4));
//GridLocalY = new CubicGrid(new int3(4, 4, 4));
//GridLocalZ = new CubicGrid(new int3(4, 4, 4));
GridAngleX = new CubicGrid(new int3(1, 1, NTilts));
GridAngleY = new CubicGrid(new int3(1, 1, NTilts));
GridAngleZ = new CubicGrid(new int3(1, 1, NTilts));
}
if (GridLocalX.Dimensions.Elements() == 1)
{
GridLocalX = new CubicGrid(new int3(4, 4, 4));
GridLocalY = new CubicGrid(new int3(4, 4, 4));
GridLocalZ = new CubicGrid(new int3(4, 4, 4));
}
//else
//{
// GridMovementX = GridMovementX.Resize(new int3(4, 4, NTilts));
// GridMovementY = GridMovementY.Resize(new int3(4, 4, NTilts));
//}
int CoarseSize = (int)Math.Round(size * ((float)CTF.PixelSize * 2 / resolution)) / 2 * 2;
int3 CoarseDims = new int3(CoarseSize, CoarseSize, 1);
// Positions the particles were extracted at/shifted to, to calculate effectively needed shifts later
float2[] ExtractedAt = new float2[NParticles * NTilts];
// Extract images, mask and resize them, create CTFs
Image ParticleImages = new Image(new int3(CoarseSize, CoarseSize, NParticles * NTilts), true, true);
Image ParticleCTFs = new Image(new int3(CoarseSize, CoarseSize, NParticles * NTilts), true);
Image ParticleWeights = null;
Image ShiftFactors = null;
#region Preflight
float KeepBFac = GlobalBfactor;
GlobalBfactor = 0;
{
Image CTFCoords = GetCTFCoords(CoarseSize, size);
#region Precalculate vectors for shifts in Fourier space
{
float2[] ShiftFactorsData = new float2[(CoarseSize / 2 + 1) * CoarseSize];
for (int y = 0; y < CoarseSize; y++)
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x;
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
ShiftFactorsData[y * (CoarseSize / 2 + 1) + x] = new float2((float)-xx / size * 2f * (float)Math.PI,
(float)-yy / size * 2f * (float)Math.PI);
}
ShiftFactors = new Image(ShiftFactorsData, new int3(CoarseSize, CoarseSize, 1), true);
}
#endregion
#region Create mask with soft edge
Image Mask;
Image MaskSubt;
{
Image MaskBig = new Image(new int3(size, size, 1));
float MaskRadius = MainWindow.Options.ExportParticleRadius / (float)CTF.PixelSize;
float SoftEdge = 16f;
float[] MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= MaskRadius)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = (float)(Math.Cos(Math.Min(1, (R - MaskRadius) / SoftEdge) * Math.PI) * 0.5 + 0.5);
}
}
//MaskBig.WriteMRC("d_maskbig.mrc");
Mask = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
Mask.RemapToFT();
MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= 30)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = 0;
}
}
MaskSubt = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
MaskSubt.RemapToFT();
MaskBig.Dispose();
}
//Mask.WriteMRC("d_masksmall.mrc");
#endregion
#region Create Fourier space mask
Image FourierMask = new Image(CoarseDims, true);
{
float[] FourierMaskData = FourierMask.GetHost(Intent.Write)[0];
int MaxR2 = CoarseSize * CoarseSize / 4;
for (int y = 0; y < CoarseSize; y++)
{
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
yy *= yy;
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x * x;
int R2 = yy + xx;
FourierMaskData[y * (CoarseSize / 2 + 1) + x] = R2 < MaxR2 ? 1 : 0;
}
}
}
#endregion
#region For each particle, create CTFs and extract & preprocess images for entire tilt series
for (int p = 0; p < NParticles; p++)
{
float3 ParticleCoords = ParticleOrigins[p];
float3[] Positions = GetPositionInImages(ParticleCoords);
float3[] ProjAngles = GetParticleAngleInImages(ParticleCoords, ParticleAngles[p]);
Image Extracted = new Image(new int3(size, size, NTilts));
float[][] ExtractedData = Extracted.GetHost(Intent.Write);
float3[] Residuals = new float3[NTilts];
Image SubtrahendsCTF = new Image(new int3(CoarseSize, CoarseSize, NTilts), true);
// Create CTFs
{
CTFStruct[] CTFParams = new CTFStruct[NTilts];
float GridStep = 1f / (NTilts - 1);
CTFStruct[] Params = new CTFStruct[NTilts];
for (int t = 0; t < NTilts; t++)
{
decimal Defocus = (decimal)Positions[t].Z;
decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
CTF CurrCTF = CTF.GetCopy();
CurrCTF.Defocus = Defocus;
CurrCTF.DefocusDelta = DefocusDelta;
CurrCTF.DefocusAngle = DefocusAngle;
CurrCTF.Scale = (decimal)Math.Cos(Angles[t] * Helper.ToRad);
CurrCTF.Bfactor = (decimal)-Dose[t] * 8;
Params[t] = CurrCTF.ToStruct();
}
GPU.CreateCTF(ParticleCTFs.GetDeviceSlice(NTilts * p, Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CoarseDims.ElementsFFT(),
Params,
false,
(uint)NTilts);
}
//{
// CTFStruct[] CTFParams = new CTFStruct[NTilts];
// float GridStep = 1f / (NTilts - 1);
// CTFStruct[] Params = new CTFStruct[NTilts];
// for (int t = 0; t < NTilts; t++)
// {
// decimal Defocus = (decimal)Positions[t].Z;
// decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
// decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
// CTF CurrCTF = CTF.GetCopy();
// CurrCTF.Defocus = Defocus;
// CurrCTF.DefocusDelta = DefocusDelta;
// CurrCTF.DefocusAngle = DefocusAngle;
// CurrCTF.Scale = 1;
// CurrCTF.Bfactor = 0;
// Params[t] = CurrCTF.ToStruct();
// }
// GPU.CreateCTF(SubtrahendsCTF.GetDevice(Intent.Write),
// CTFCoords.GetDevice(Intent.Read),
// (uint)CoarseDims.ElementsFFT(),
// Params,
// false,
// (uint)NTilts);
//}
// Extract images
{
for (int t = 0; t < NTilts; t++)
{
ExtractedAt[p * NTilts + t] = new float2(Positions[t].X, Positions[t].Y);
Positions[t] -= size / 2;
int2 IntPosition = new int2((int)Positions[t].X, (int)Positions[t].Y);
float2 Residual = new float2(-(Positions[t].X - IntPosition.X), -(Positions[t].Y - IntPosition.Y));
Residuals[t] = new float3(Residual / size * CoarseSize);
float[] OriginalData;
lock (tiltStack)
OriginalData = tiltStack.GetHost(Intent.Read)[t];
float[] ImageData = ExtractedData[t];
for (int y = 0; y < size; y++)
{
int PosY = (y + IntPosition.Y + tiltStack.Dims.Y) % tiltStack.Dims.Y;
for (int x = 0; x < size; x++)
{
int PosX = (x + IntPosition.X + tiltStack.Dims.X) % tiltStack.Dims.X;
ImageData[y * size + x] = OriginalData[PosY * tiltStack.Dims.X + PosX];
}
}
}
GPU.NormParticles(Extracted.GetDevice(Intent.Read),
Extracted.GetDevice(Intent.Write),
new int3(size, size, 1),
(uint)(MainWindow.Options.ExportParticleRadius / CTF.PixelSize),
true,
(uint)NTilts);
Image Scaled = Extracted.AsScaled(new int2(CoarseSize, CoarseSize));
//Scaled.WriteMRC("d_scaled.mrc");
Extracted.Dispose();
Scaled.ShiftSlices(Residuals);
Scaled.RemapToFT();
//GPU.NormalizeMasked(Scaled.GetDevice(Intent.Read),
// Scaled.GetDevice(Intent.Write),
// MaskSubt.GetDevice(Intent.Read),
// (uint)Scaled.ElementsSliceReal,
// (uint)NTilts);
//{
// //Image SubtrahendsFT = subtrahendReference.Project(new int2(CoarseSize, CoarseSize), ProjAngles, CoarseSize / 2);
// //SubtrahendsFT.Multiply(SubtrahendsCTF);
// //Image Subtrahends = SubtrahendsFT.AsIFFT();
// //SubtrahendsFT.Dispose();
// ////GPU.NormalizeMasked(Subtrahends.GetDevice(Intent.Read),
// //// Subtrahends.GetDevice(Intent.Write),
// //// MaskSubt.GetDevice(Intent.Read),
// //// (uint)Subtrahends.ElementsSliceReal,
// //// (uint)NTilts);
// //Scaled.Subtract(Subtrahends);
// //Subtrahends.Dispose();
// Image FocusMaskFT = maskReference.Project(new int2(CoarseSize, CoarseSize), ProjAngles, CoarseSize / 2);
// Image FocusMask = FocusMaskFT.AsIFFT();
// FocusMaskFT.Dispose();
// Scaled.Multiply(FocusMask);
// FocusMask.Dispose();
//}
Scaled.MultiplySlices(Mask);
GPU.FFT(Scaled.GetDevice(Intent.Read),
ParticleImages.GetDeviceSlice(p * NTilts, Intent.Write),
CoarseDims,
(uint)NTilts);
Scaled.Dispose();
SubtrahendsCTF.Dispose();
}
}
#endregion
ParticleCTFs.MultiplySlices(FourierMask);
Mask.Dispose();
FourierMask.Dispose();
MaskSubt.Dispose();
Image ParticleCTFsAbs = new Image(ParticleCTFs.GetDevice(Intent.Read), ParticleCTFs.Dims, true);
ParticleCTFsAbs.Abs();
ParticleWeights = ParticleCTFsAbs.AsSum2D();
ParticleCTFsAbs.Dispose();
{
float[] ParticleWeightsData = ParticleWeights.GetHost(Intent.ReadWrite)[0];
float Max = MathHelper.Max(ParticleWeightsData);
for (int i = 0; i < ParticleWeightsData.Length; i++)
ParticleWeightsData[i] /= Max;
}
CTFCoords.Dispose();
//Image CheckImages = ParticleImages.AsIFFT();
//CheckImages.WriteMRC("d_particleimages.mrc");
//CheckImages.Dispose();
//ParticleCTFs.WriteMRC("d_particlectfs.mrc");
}
GlobalBfactor = KeepBFac;
#endregion
bool DoPerParticleMotion = true;
bool DoImageAlignment = true;
#region BFGS evaluation and gradient
double[] StartParams;
Func<double[], Tuple<float2[], float3[]>> GetImageShiftsAndAngles;
Func<double[], float2[]> GetImageShifts;
Func<float3[], Image> GetProjections;
Func<double[], double[]> EvalIndividual;
Func<double[], double> Eval;
Func<double[], double[]> Gradient;
{
List<double> StartParamsList = new List<double>();
StartParamsList.AddRange(CreateVectorFromGrids(Dimensions.X));
StartParamsList.AddRange(CreateVectorFromParameters(ParticleOrigins, ParticleOrigins2, ParticleAngles, ParticleAngles2, size));
StartParams = StartParamsList.ToArray();
// Remember where the values for each grid are stored in the optimized vector
List<Tuple<int, int>> VectorGridRanges = new List<Tuple<int, int>>();
List<int> GridElements = new List<int>();
List<int> GridSliceElements = new List<int>();
{
int Start = 0;
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridMovementX.Dimensions.Elements()));
Start += (int)GridMovementX.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridMovementY.Dimensions.Elements()));
Start += (int)GridMovementY.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridAngleX.Dimensions.Elements()));
Start += (int)GridAngleX.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridAngleY.Dimensions.Elements()));
Start += (int)GridAngleY.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridAngleZ.Dimensions.Elements()));
Start += (int)GridAngleZ.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridLocalX.Dimensions.Elements()));
Start += (int)GridLocalX.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridLocalY.Dimensions.Elements()));
Start += (int)GridLocalY.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridLocalZ.Dimensions.Elements()));
GridElements.Add((int)GridMovementX.Dimensions.Elements());
GridElements.Add((int)GridMovementY.Dimensions.Elements());
GridElements.Add((int)GridAngleX.Dimensions.Elements());
GridElements.Add((int)GridAngleY.Dimensions.Elements());
GridElements.Add((int)GridAngleZ.Dimensions.Elements());
GridElements.Add((int)GridLocalX.Dimensions.Elements());
GridElements.Add((int)GridLocalY.Dimensions.Elements());
GridElements.Add((int)GridLocalZ.Dimensions.Elements());
GridSliceElements.Add((int)GridMovementX.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridMovementY.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridAngleX.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridAngleY.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridAngleZ.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridLocalX.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridLocalY.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridLocalZ.Dimensions.ElementsSlice());
}
int NVectorGridParams = VectorGridRanges.Last().Item2;
int NVectorParticleParams = NParticles * 12;
GetImageShiftsAndAngles = input =>
{
// Retrieve particle positions & angles, and grids from input vector
float3[] NewPositions, NewPositions2, NewAngles, NewAngles2;
GetParametersFromVector(input, NParticles, size, out NewPositions, out NewPositions2, out NewAngles, out NewAngles2);
SetGridsFromVector(input, Dimensions.X);
// Using current positions, angles and grids, get parameters for image shifts and reference projection angles
float2[] ImageShifts = new float2[NParticles * NTilts];
float3[] ImageAngles = new float3[NParticles * NTilts];
float3[] PerTiltPositions = new float3[NParticles * NTilts];
float3[] PerTiltAngles = new float3[NParticles * NTilts];
int[] SortedDosePrecalc = IndicesSortedDose;
for (int p = 0; p < NParticles; p++)
{
if (DoPerParticleMotion)
{
float3 CoordsDiff = NewPositions2[p] - NewPositions[p];
float3 AnglesDiff = NewAngles2[p] - NewAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[p * NTilts + t] = NewPositions[p] + CoordsDiff * DoseID;
PerTiltAngles[p * NTilts + t] = NewAngles[p] + AnglesDiff * DoseID;
}
}
else
{
for (int t = 0; t < NTilts; t++)
{
PerTiltPositions[p * NTilts + t] = NewPositions[p];
PerTiltAngles[p * NTilts + t] = NewAngles[p];
}
}
}
float3[] CurrPositions = GetPositionInImages(PerTiltPositions);
float3[] CurrAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
for (int i = 0; i < ImageShifts.Length; i++)
{
ImageShifts[i] = new float2(ExtractedAt[i].X - CurrPositions[i].X,
ExtractedAt[i].Y - CurrPositions[i].Y); // -diff because those are extraction positions, i. e. opposite direction of shifts
ImageAngles[i] = CurrAngles[i];
}
return new Tuple<float2[], float3[]>(ImageShifts, ImageAngles);
};
GetImageShifts = input =>
{
// Retrieve particle positions & angles, and grids from input vector
float3[] NewPositions, NewPositions2, NewAngles, NewAngles2;
GetParametersFromVector(input, NParticles, size, out NewPositions, out NewPositions2, out NewAngles, out NewAngles2);
SetGridsFromVector(input, Dimensions.X);
// Using current positions, angles and grids, get parameters for image shifts and reference projection angles
float2[] ImageShifts = new float2[NParticles * NTilts];
float3[] PerTiltPositions = new float3[NParticles * NTilts];
int[] SortedDosePrecalc = IndicesSortedDose;
for (int p = 0; p < NParticles; p++)
{
if (DoPerParticleMotion)
{
float3 CoordsDiff = NewPositions2[p] - NewPositions[p];
float3 AnglesDiff = NewAngles2[p] - NewAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[p * NTilts + t] = NewPositions[p] + CoordsDiff * DoseID;
}
}
else
{
for (int t = 0; t < NTilts; t++)
PerTiltPositions[p * NTilts + t] = NewPositions[p];
}
}
float3[] CurrPositions = GetPositionInImages(PerTiltPositions);
for (int i = 0; i < ImageShifts.Length; i++)
ImageShifts[i] = new float2(ExtractedAt[i].X - CurrPositions[i].X,
ExtractedAt[i].Y - CurrPositions[i].Y); // -diff because those are extraction positions, i. e. opposite direction of shifts
return ImageShifts;
};
GetProjections = imageAngles =>
{
Image Projections = new Image(IntPtr.Zero, new int3(CoarseSize, CoarseSize, NParticles * NTilts), true, true);
foreach (var subset in SubsetRanges)
{
Projector Reference = references[subset.Key];
int SubsetStart = subset.Value.Item1 * NTilts;
int SubsetEnd = subset.Value.Item2 * NTilts;
float3[] SubsetAngles = imageAngles.Skip(SubsetStart).Take(SubsetEnd - SubsetStart).ToArray();
GPU.ProjectForward(Reference.Data.GetDevice(Intent.Read),
Projections.GetDeviceSlice(SubsetStart, Intent.Write),
Reference.Data.Dims,
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(SubsetAngles),
Reference.Oversampling,
(uint)(SubsetEnd - SubsetStart));
}
/*Image CheckProjections = Projections.AsIFFT();
//CheckProjections.RemapFromFT();
CheckProjections.WriteMRC("d_projections.mrc");
CheckProjections.Dispose();*/
return Projections;
};
EvalIndividual = input =>
{
Tuple<float2[], float3[]> ShiftsAndAngles = GetImageShiftsAndAngles(input);
Image Projections = GetProjections(ShiftsAndAngles.Item2);
float[] Results = new float[NParticles * NTilts];
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsAndAngles.Item1),
Results,
(uint)(NParticles * NTilts));
Projections.Dispose();
return Results.Select(i => (double)i).ToArray();
};
int OptimizationIterations = 0;
bool GetOut = false;
double Delta = 0.1;
float Delta2 = 2 * (float)Delta;
int[] WarpGridIDs = { 5, 6, 7 };
Dictionary<int, float2[][]> WiggleWeightsWarp = new Dictionary<int, float2[][]>();
foreach (var gridID in WarpGridIDs)
{
int NElements = GridElements[gridID];
WiggleWeightsWarp.Add(gridID, new float2[NElements][]);
for (int ge = 0; ge < NElements; ge++)
{
double[] InputMinus = new double[StartParams.Length], InputPlus = new double[StartParams.Length];
for (int i = 0; i < StartParams.Length; i++)
{
InputMinus[i] = StartParams[i];
InputPlus[i] = StartParams[i];
}
InputMinus[VectorGridRanges[gridID].Item1 + ge] -= Delta;
InputPlus[VectorGridRanges[gridID].Item1 + ge] += Delta;
float2[] ImageShiftsPlus = GetImageShifts(InputPlus);
float2[] ImageShiftsMinus = GetImageShifts(InputMinus);
float2[] Weights = new float2[ImageShiftsPlus.Length];
for (int i = 0; i < ImageShiftsPlus.Length; i++)
Weights[i] = (ImageShiftsPlus[i] - ImageShiftsMinus[i]) / Delta2;
WiggleWeightsWarp[gridID][ge] = Weights;
}
}
Eval = input =>
{
double Result = EvalIndividual(input).Sum();
lock (tableIn)
Debug.WriteLine(GPU.GetDevice() + ", " + RootName + ": " + Result);
OptimizationIterations++;
return Result;
};
Func<double[], double[], double, double[]> GradientParticles = (inputMinus, inputPlus, delta) =>
{
double[] EvalMinus = EvalIndividual(inputMinus);
double[] EvalPlus = EvalIndividual(inputPlus);
double[] Diff = new double[EvalMinus.Length];
for (int i = 0; i < Diff.Length; i++)
Diff[i] = (EvalPlus[i] - EvalMinus[i]) / (2 * delta);
return Diff;
};
Gradient = input =>
{
double[] Result = new double[input.Length];
if (OptimizationIterations > 60)
return Result;
float2[] ImageShiftGradients = new float2[NParticles * NTilts];
#region Compute gradient for individual image shifts
{
Tuple<float2[], float3[]> ShiftsAndAngles = GetImageShiftsAndAngles(input);
Image Projections = GetProjections(ShiftsAndAngles.Item2);
float2[] ShiftsXPlus = new float2[NParticles * NTilts];
float2[] ShiftsXMinus = new float2[NParticles * NTilts];
float2[] ShiftsYPlus = new float2[NParticles * NTilts];
float2[] ShiftsYMinus = new float2[NParticles * NTilts];
float2 DeltaX = new float2((float)Delta, 0);
float2 DeltaY = new float2(0, (float)Delta);
for (int i = 0; i < ShiftsXPlus.Length; i++)
{
ShiftsXPlus[i] = ShiftsAndAngles.Item1[i] + DeltaX;
ShiftsXMinus[i] = ShiftsAndAngles.Item1[i] - DeltaX;
ShiftsYPlus[i] = ShiftsAndAngles.Item1[i] + DeltaY;
ShiftsYMinus[i] = ShiftsAndAngles.Item1[i] - DeltaY;
}
float[] ScoresXPlus = new float[NParticles * NTilts];
float[] ScoresXMinus = new float[NParticles * NTilts];
float[] ScoresYPlus = new float[NParticles * NTilts];
float[] ScoresYMinus = new float[NParticles * NTilts];
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsXPlus),
ScoresXPlus,
(uint)(NParticles * NTilts));
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsXMinus),
ScoresXMinus,
(uint)(NParticles * NTilts));
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsYPlus),
ScoresYPlus,
(uint)(NParticles * NTilts));
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsYMinus),
ScoresYMinus,
(uint)(NParticles * NTilts));
Projections.Dispose();
for (int i = 0; i < ImageShiftGradients.Length; i++)
{
ImageShiftGradients[i] = new float2((ScoresXPlus[i] - ScoresXMinus[i]) / Delta2,
(ScoresYPlus[i] - ScoresYMinus[i]) / Delta2);
}
}
#endregion
// First, do particle parameters, i. e. 3D position within tomogram, rotation, across 2 points in time
// Altering each particle's parameters results in a change in its NTilts images, but nothing else
{
int[] TranslationIDs = DoPerParticleMotion ? new[] { 0, 1, 2, 3, 4, 5 } : new[] { 0, 1, 2 };
int[] RotationIDs = DoPerParticleMotion ? new [] {6, 7, 8, 9, 10, 11} : new [] { 6, 7, 8};
foreach (var paramID in RotationIDs)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int p = 0; p < NParticles; p++)
{
InputMinus[NVectorGridParams + p * 12 + paramID] -= Delta;
InputPlus[NVectorGridParams + p * 12 + paramID] += Delta;
}
double[] ResultParticles = GradientParticles(InputMinus, InputPlus, Delta);
for (int p = 0; p < NParticles; p++)
{
double ParticleSum = 0;
for (int t = 0; t < NTilts; t++)
ParticleSum += ResultParticles[p * NTilts + t];
Result[NVectorGridParams + p * 12 + paramID] = ParticleSum;
}
}
// Translation-related gradients can all be computed efficiently from previously retrieved per-image gradients
foreach (var paramID in TranslationIDs)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int p = 0; p < NParticles; p++)
{
InputMinus[NVectorGridParams + p * 12 + paramID] -= Delta;
InputPlus[NVectorGridParams + p * 12 + paramID] += Delta;
}
float2[] ImageShiftsPlus = GetImageShifts(InputPlus);
float2[] ImageShiftsMinus = GetImageShifts(InputMinus);
for (int p = 0; p < NParticles; p++)
{
double ParticleSum = 0;
for (int t = 0; t < NTilts; t++)
{
int i = p * NTilts + t;
float2 ShiftDelta = (ImageShiftsPlus[i] - ImageShiftsMinus[i]) / Delta2;
float ShiftGradient = ShiftDelta.X * ImageShiftGradients[i].X + ShiftDelta.Y * ImageShiftGradients[i].Y;
ParticleSum += ShiftGradient;
}
Result[NVectorGridParams + p * 12 + paramID] = ParticleSum;
}
}
// If there is no per-particle motion, just copy the gradients for these parameters from parameterIDs 0-5
if (!DoPerParticleMotion)
{
int[] RedundantIDs = { 3, 4, 5, 9, 10, 11 };
foreach (var paramID in RedundantIDs)
for (int p = 0; p < NParticles; p++)
Result[NVectorGridParams + p * 12 + paramID] = Result[NVectorGridParams + p * 12 + paramID - 3];
}
}
// Now deal with grids. Each grid slice (i. e. temporal point) will correspond to one tilt only, thus the gradient
// for each slice is the (weighted, in case of spatial resolution) sum of NParticles images in the corresponding tilt.
if (DoImageAlignment)
{
int[] RotationGridIDs = { 2, 3, 4 };
foreach (var gridID in RotationGridIDs)
{
int SliceElements = GridSliceElements[gridID];
for (int se = 0; se < SliceElements; se++)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int gp = VectorGridRanges[gridID].Item1 + se; gp < VectorGridRanges[gridID].Item2; gp += SliceElements)
{
InputMinus[gp] -= Delta;
InputPlus[gp] += Delta;
}
double[] ResultParticles = GradientParticles(InputMinus, InputPlus, Delta);
for (int i = 0; i < ResultParticles.Length; i++)
{
int GridTime = i % NTilts;
Result[VectorGridRanges[gridID].Item1 + GridTime * SliceElements + se] += ResultParticles[i];
}
}
}
// Translation-related gradients can all be computed efficiently from previously retrieved per-image gradients
int[] TranslationGridIDs = { 0, 1 };
foreach (var gridID in TranslationGridIDs)
{
int SliceElements = GridSliceElements[gridID];
for (int se = 0; se < SliceElements; se++)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int gp = VectorGridRanges[gridID].Item1 + se; gp < VectorGridRanges[gridID].Item2; gp += SliceElements)
{
InputMinus[gp] -= Delta;
InputPlus[gp] += Delta;
}
float2[] ImageShiftsPlus = GetImageShifts(InputPlus);
float2[] ImageShiftsMinus = GetImageShifts(InputMinus);
for (int i = 0; i < ImageShiftsPlus.Length; i++)
{
float2 ShiftDelta = (ImageShiftsPlus[i] - ImageShiftsMinus[i]) / Delta2;
float ShiftGradient = ShiftDelta.X * ImageShiftGradients[i].X + ShiftDelta.Y * ImageShiftGradients[i].Y;
int GridSlice = i % NTilts;
Result[VectorGridRanges[gridID].Item1 + GridSlice * SliceElements + se] += ShiftGradient;
}
}
}
// Warp grids don't have any shortcuts for getting multiple gradients at once, so they use pre-calculated wiggle weights
foreach (var gridID in WarpGridIDs)
{
int NElements = GridElements[gridID];
for (int ge = 0; ge < NElements; ge++)
{
float2[] Weights = WiggleWeightsWarp[gridID][ge];
for (int i = 0; i < Weights.Length; i++)
{
float2 ShiftDelta = Weights[i];
float ShiftGradient = ShiftDelta.X * ImageShiftGradients[i].X + ShiftDelta.Y * ImageShiftGradients[i].Y;
Result[VectorGridRanges[gridID].Item1 + ge] += ShiftGradient;
}
}
}
}
return Result;
};
}
#endregion
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Gradient);
Optimizer.Epsilon = 3e-7;
Optimizer.Maximize(StartParams);
float3[] OptimizedOrigins, OptimizedOrigins2, OptimizedAngles, OptimizedAngles2;
GetParametersFromVector(StartParams, NParticles, size, out OptimizedOrigins, out OptimizedOrigins2, out OptimizedAngles, out OptimizedAngles2);
SetGridsFromVector(StartParams, Dimensions.X);
#region Calculate correlation scores, update table with new positions and angles
{
double[] ImageScores = EvalIndividual(StartParams);
float[] ParticleScores = new float[NParticles];
for (int i = 0; i < ImageScores.Length; i++)
ParticleScores[i / NTilts] += (float)ImageScores[i];
//if (!tableIn.HasColumn("rlnOriginXPrior"))
// tableIn.AddColumn("rlnOriginXPrior");
//if (!tableIn.HasColumn("rlnOriginYPrior"))
// tableIn.AddColumn("rlnOriginYPrior");
//if (!tableIn.HasColumn("rlnOriginZPrior"))
// tableIn.AddColumn("rlnOriginZPrior");
//if (!tableIn.HasColumn("rlnAngleRotPrior"))
// tableIn.AddColumn("rlnAngleRotPrior");
//if (!tableIn.HasColumn("rlnAngleTiltPrior"))
// tableIn.AddColumn("rlnAngleTiltPrior");
//if (!tableIn.HasColumn("rlnAnglePsiPrior"))
// tableIn.AddColumn("rlnAnglePsiPrior");
lock (tableIn)
for (int p = 0; p < NParticles; p++)
{
int Row = RowIndices[SubsetContinuousIDs[p]];
tableIn.SetRowValue(Row, "rlnCoordinateX", OptimizedOrigins[p].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnCoordinateY", OptimizedOrigins[p].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnCoordinateZ", OptimizedOrigins[p].Z.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnOriginXPrior", OptimizedOrigins2[p].X.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnOriginYPrior", OptimizedOrigins2[p].Y.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnOriginZPrior", OptimizedOrigins2[p].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnAngleRot", OptimizedAngles[p].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnAngleTilt", OptimizedAngles[p].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnAnglePsi", OptimizedAngles[p].Z.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnAngleRotPrior", OptimizedAngles2[p].X.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnAngleTiltPrior", OptimizedAngles2[p].Y.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnAnglePsiPrior", OptimizedAngles2[p].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnParticleSelectZScore", ParticleScores[p].ToString(CultureInfo.InvariantCulture));
}
}
#endregion
ParticleImages?.Dispose();
ParticleCTFs?.Dispose();
ParticleWeights?.Dispose();
ShiftFactors?.Dispose();
#region Extract particles at full resolution and back-project them into the reconstruction volumes
{
GPU.SetDevice(0);
Image CTFCoords = GetCTFCoords(size, size);
int[] SortedDosePrecalc = IndicesSortedDose;
foreach (var subsetRange in SubsetRanges)
{
lock (outReconstructions[subsetRange.Key])
{
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3[] PerTiltPositions = new float3[NTilts];
float3[] PerTiltAngles = new float3[NTilts];
float3 CoordsDiff = OptimizedOrigins2[p] - OptimizedOrigins[p];
float3 AnglesDiff = OptimizedAngles2[p] - OptimizedAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[t] = OptimizedOrigins[p] + CoordsDiff * DoseID;
PerTiltAngles[t] = OptimizedAngles[p] + AnglesDiff * DoseID;
}
Image FullParticleImages = GetSubtomoImages(tiltStack, size, PerTiltPositions, true);
Image FullParticleCTFs = GetSubtomoCTFs(PerTiltPositions, CTFCoords);
FullParticleImages.Multiply(FullParticleCTFs);
FullParticleCTFs.Abs();
float3[] FullParticleAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
outReconstructions[subsetRange.Key].BackProject(FullParticleImages, FullParticleCTFs, FullParticleAngles);
FullParticleImages.Dispose();
FullParticleCTFs.Dispose();
}
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3[] PerTiltPositions = new float3[NTilts];
float3[] PerTiltAngles = new float3[NTilts];
float3 CoordsDiff = OptimizedOrigins2[p] - OptimizedOrigins[p];
float3 AnglesDiff = OptimizedAngles2[p] - OptimizedAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[t] = OptimizedOrigins[p] + CoordsDiff * DoseID;
PerTiltAngles[t] = OptimizedAngles[p] + AnglesDiff * DoseID;
}
float3[] FullParticleAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
Image FullParticleCTFs = GetSubtomoCTFs(PerTiltPositions, CTFCoords, false);
Image FullParticleCTFWeights = GetSubtomoCTFs(PerTiltPositions, CTFCoords, true);
// CTF has to be converted to complex numbers with imag = 0
float2[] CTFsComplexData = new float2[FullParticleCTFs.ElementsComplex];
float[] CTFWeightsData = new float[FullParticleCTFs.ElementsComplex];
float[] CTFsContinuousData = FullParticleCTFs.GetHostContinuousCopy();
float[] CTFWeightsContinuousData = FullParticleCTFWeights.GetHostContinuousCopy();
for (int i = 0; i < CTFsComplexData.Length; i++)
{
CTFsComplexData[i] = new float2(Math.Abs(CTFsContinuousData[i] * CTFWeightsContinuousData[i]), 0);
CTFWeightsData[i] = Math.Abs(CTFWeightsContinuousData[i]);
}
Image CTFsComplex = new Image(CTFsComplexData, FullParticleCTFs.Dims, true);
Image CTFWeights = new Image(CTFWeightsData, FullParticleCTFs.Dims, true);
outCTFReconstructions[subsetRange.Key].BackProject(CTFsComplex, CTFWeights, FullParticleAngles);
FullParticleCTFs.Dispose();
FullParticleCTFWeights.Dispose();
CTFsComplex.Dispose();
CTFWeights.Dispose();
}
outReconstructions[subsetRange.Key].FreeDevice();
outCTFReconstructions[subsetRange.Key].FreeDevice();
}
}
CTFCoords.Dispose();
}
#endregion
SaveMeta();
}
public static void FitCTF(float2[] data, Func<float[], float[]> approximation, float[] zeros, float[] peaks, out Cubic1D background, out Cubic1D scale)
{
float MinX = MathHelper.Min(data.Select(p => p.X)), MaxX = MathHelper.Max(data.Select(p => p.X)), ScaleX = 1f / (MaxX - MinX);
float MinY = MathHelper.Min(data.Select(p => p.Y)), MaxY = MathHelper.Max(data.Select(p => p.Y)), ScaleY = 1f / (MaxY - MinY);
if (float.IsNaN(ScaleY))
ScaleY = 1f;
peaks = peaks.Where(v => v >= MinX && v <= MaxX).Where((v, i) => i % 1 == 0).ToArray();
zeros = zeros.Where(v => v >= MinX && v <= MaxX).Where((v, i) => i % 1 == 0).ToArray();
float2[] ScaledData = data.Select(p => new float2((p.X - MinX) * ScaleX, (p.Y - MinY) * ScaleY)).ToArray();
float StdY = MathHelper.StdDev(data.Select(p => p.Y).ToArray());
double[] Start = new double[zeros.Length + peaks.Length];
double[] NodeX = new double[zeros.Length + peaks.Length];
Cubic1D DataSpline = new Cubic1D(data);
for (int i = 0; i < zeros.Length; i++)
{
NodeX[i] = (zeros[i] - MinX) * ScaleX;
Start[i] = DataSpline.Interp(zeros[i]) - MinY;
}
{
Cubic1D PreliminaryBackground = new Cubic1D(Helper.Zip(NodeX.Take(zeros.Length).Select(v => (float)v).ToArray(),
Start.Take(zeros.Length).Select(v => (float)v).ToArray()));
float[] PreliminaryInterpolated = PreliminaryBackground.Interp(data.Select(v => (v.X - MinX) * ScaleX).ToArray());
float2[] BackgroundSubtracted = data.Select((v, i) => new float2(v.X, v.Y - MinY - PreliminaryInterpolated[i])).ToArray();
Cubic1D BackgroundSpline = new Cubic1D(BackgroundSubtracted);
for (int i = 0; i < peaks.Length; i++)
{
NodeX[i + zeros.Length] = (peaks[i] - MinX) * ScaleX;
float PeakValue = BackgroundSpline.Interp(peaks[i]);
Start[i + zeros.Length] = Math.Max(0.0001f, PeakValue);
}
}
float[] DataX = ScaledData.Select(p => p.X).ToArray();
float[] OriginalDataX = data.Select(p => p.X).ToArray();
float[] SimulatedCTF = approximation(OriginalDataX);
float2[] NodesBackground = new float2[zeros.Length];
for (int i = 0; i < NodesBackground.Length; i++)
NodesBackground[i] = new float2((float)NodeX[i], 0f);
float2[] NodesScale = new float2[peaks.Length];
for (int i = 0; i < NodesScale.Length; i++)
NodesScale[i] = new float2((float)NodeX[i + zeros.Length], 0f);
Func<double[], double> Eval = input =>
{
float2[] NodesBackgroundCopy = new float2[NodesBackground.Length];
for (int i = 0; i < zeros.Length; i++)
NodesBackgroundCopy[i] = new float2(NodesBackground[i].X, (float)input[i]);
float2[] NodesScaleCopy = new float2[NodesScale.Length];
for (int i = 0; i < peaks.Length; i++)
NodesScaleCopy[i] = new float2(NodesScale[i].X, (float)input[i + zeros.Length]);
float[] InterpolatedBackground = new Cubic1D(NodesBackgroundCopy).Interp(DataX);
float[] InterpolatedScale = new Cubic1D(NodesScaleCopy).Interp(DataX);
double Sum = 0f;
for (int i = 0; i < ScaledData.Length; i++)
{
double Diff = ScaledData[i].Y - (InterpolatedBackground[i] + SimulatedCTF[i] * (double)InterpolatedScale[i]) * ScaleY;
Sum += Diff * Diff;
if (InterpolatedScale[i] < 0.0005f)
Sum += (0.0005 - InterpolatedScale[i]) * 10;
}
//return Math.Sqrt(Sum / data.Length) * 10;
return 0;
};
Func<double[], double[]> Gradient = input =>
{
double[] Result = new double[input.Length];
//Parallel.For(0, input.Length, i =>
for (int i = 0; i < input.Length; i++)
{
double[] UpperInput = new double[input.Length];
input.CopyTo(UpperInput, 0);
UpperInput[i] += 0.0001;
double UpperValue = Eval(UpperInput);
double[] LowerInput = new double[input.Length];
input.CopyTo(LowerInput, 0);
LowerInput[i] -= 0.0001;
double LowerValue = Eval(LowerInput);
Result[i] = (UpperValue - LowerValue) / 0.0002;
}//);
return Result;
};
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(Start.Length, Eval, Gradient);
Optimizer.Minimize(Start);
{
for (int i = 0; i < zeros.Length; i++)
NodesBackground[i] = new float2((float) NodeX[i] / ScaleX + MinX, (float) Optimizer.Solution[i] + MinY);
for (int i = 0; i < peaks.Length; i++)
NodesScale[i] = new float2((float)NodeX[i + zeros.Length] / ScaleX + MinX, Math.Max(0.001f, (float)Optimizer.Solution[i + zeros.Length]));
background = new Cubic1D(NodesBackground);
scale = new Cubic1D(NodesScale);
}
}
private unsafe int cvsmod(double[] s, ref double stp, ref int info,
ref int nfev, double[] wa, ref double dginit, ref double dgout)
{
int n = numberOfVariables;
if (info == 1)
{
goto L321;
}
infoc = 1;
if (stp <= 0) // Check the input parameters for errors
{
throw new LineSearchFailedException(1, "Invalid step size.");
}
// Compute the initial gradient in the search direction
// and check that S is a descent direction.
if (dginit >= 0)
{
throw new LineSearchFailedException(0, "The search direction is not a descent direction.");
}
// Initialize local variables
brackt = false;
stage1 = true;
nfev = 0;
finit = f;
dgtest = ftol * dginit;
width = stpmax - stpmin;
width1 = width / 0.5;
for (int j = 0; j < x.Length; ++j)
{
wa[j] = x[j];
}
// The variables STX, FX, DGX contain the values of the step,
// function, and directional derivative at the best step.
// The variables STY, FY, DGY contain the value of the step,
// function, and derivative at the other endpoint of the interval
// of uncertainty.
// The variables STP, F, DG contain the values of the step,
// function, and derivative at the current step.
stx = 0;
fx = finit;
dgx = dginit;
sty = 0;
fy = finit;
dgy = dginit;
L30: // Start of iteration.
// Set the minimum and maximum steps to correspond
// to the present interval of uncertainty.
if (brackt)
{
stmin = Math.Min(stx, sty);
stmax = Math.Max(stx, sty);
}
else
{
stmin = stx;
stmax = stp + 4 * (stp - stx);
}
// Force the step to be within
// the bounds STPMAX and STPMIN.
stp = Math.Max(stp, stpmin);
stp = Math.Min(stp, stpmax);
// If an unusual termination is to occur then
// let STP be the lowest point obtained so far.
if (brackt && (stp <= stmin || stp >= stmax) || nfev >= maxfev - 1 ||
infoc == 0 || brackt && stmax - stmin <= xtol * stmax)
{
stp = stx;
}
// Evaluate the function and gradient at STP
// and compute the directional derivative.
for (int j = 0; j < s.Length; ++j)
{
x[j] = wa[j] + stp * s[j];
}
// Fetch function and gradient
f = Function(x);
g = Gradient(x);
info = 0;
nfev++;
dg2 = 0;
for (int j = 0; j < g.Length; ++j)
{
dg2 += g[j] * s[j];
}
ftest1 = finit + stp * dgtest;
if ((brackt && (stp <= stmin || stp >= stmax)) || infoc == 0)
{
throw new LineSearchFailedException(6, "Rounding errors prevent further progress." +
"There may not be a step which satisfies the sufficient decrease and curvature conditions. Tolerances may be too small.");
}
if (stp == stpmax && f <= ftest1 && dg2 <= dgtest)
{
throw new LineSearchFailedException(5, "The step size has reached the upper bound.");
}
if (stp == stpmin && (f > ftest1 || dg2 >= dgtest))
{
throw new LineSearchFailedException(4, "The step size has reached the lower bound.");
}
if (nfev >= maxfev)
{
throw new LineSearchFailedException(3, "Maximum number of function evaluations has been reached.");
}
if (brackt && stmax - stmin <= xtol * stmax)
{
throw new LineSearchFailedException(2, "Relative width of the interval of uncertainty is at machine precision.");
}
// More's code has been modified so that at least one new
// function value is computed during the line search (enforcing
// at least one interpolation is not easy, since the code may
// override an interpolation)
if (f <= ftest1 && Math.Abs(dg2) <= gtol * (-dginit) && nfev > 1)
{
info = 1;
dgout = dg2;
return(0);
}
L321:
// In the first stage we seek a step for which the modified
// function has a nonpositive value and nonnegative derivative.
if (stage1 && f <= ftest1 && dg2 >= Math.Min(ftol, gtol) * dginit)
{
stage1 = false;
}
// A modified function is used to predict the step only if
// we have not obtained a step for which the modified function
// has a nonpositive function value and nonnegative derivative,
// and if a lower function value has been obtained but the
// decrease is not sufficient.
if (stage1 && f <= fx && f > ftest1)
{
// Define the modified function and derivative values
double fm = f - stp * dgtest;
double fxm = fx - stx * dgtest;
double fym = fy - sty * dgtest;
double dgm = dg2 - dgtest;
double dgxm = dgx - dgtest;
double dgym = dgy - dgtest;
// Call CSTEPM to update the interval of
// uncertainty and to compute the new step.
BroydenFletcherGoldfarbShanno.SearchStep(ref stx, ref fxm, ref dgxm,
ref sty, ref fym, ref dgym, ref stp, fm, dgm, ref brackt, out infoc);
// Reset the function and gradient values for f.
fx = fxm + stx * dgtest;
fy = fym + sty * dgtest;
dgx = dgxm + dgtest;
dgy = dgym + dgtest;
}
else
{
// Call CSTEPM to update the interval of
// uncertainty and to compute the new step.
BroydenFletcherGoldfarbShanno.SearchStep(ref stx, ref fx, ref dgx,
ref sty, ref fy, ref dgy, ref stp, f, dg2, ref brackt, out infoc);
}
// Force a sufficient decrease in the
// size of the interval of uncertainty.
if (brackt)
{
if ((Math.Abs(sty - stx)) >= 0.66 * width1)
{
stp = stx + 0.5 * (sty - stx);
}
width1 = width;
width = Math.Abs(sty - stx);
}
goto L30;
}
private void OptimizePerTiltWeights()
{
if (!Options.Movies.Any(m => m.GetType() == typeof(TiltSeries)))
return;
Image Mask = StageDataLoad.LoadMap("F:\\stefanribo\\vlion\\mask_warped2_OST_post.mrc", new int2(1, 1), 0, typeof(float));
List<Image> SubsetMasks = new List<Image> { Mask, Mask };
int3 Dims = Mask.Dims;
float AngleMin = float.MaxValue, AngleMax = float.MinValue;
float DoseMax = float.MinValue;
List<WeightOptContainer> Reconstructions = new List<WeightOptContainer>();
Dictionary<TiltSeries, int> SeriesIndices = new Dictionary<TiltSeries, int>();
int NTilts = 0;
{
TiltSeries Series = (TiltSeries)Options.Movies[0];
string[] FileNames = Directory.EnumerateFiles(Series.WeightOptimizationDir, "subset*.mrc").Where(p => p.Contains("subset3") || p.Contains("subset4")).Select(v => new FileInfo(v).Name).ToArray();
string[] MapNames = FileNames.Where(v => !v.Contains(".weight.mrc")).ToArray();
string[] WeightNames = FileNames.Where(v => v.Contains(".weight.mrc")).ToArray();
if (MapNames.Length != WeightNames.Length)
throw new Exception("Number of reconstructions and weights does not match!");
string[] MapSuffixes = MapNames;
int[] MapSubsets = MapSuffixes.Select(v =>
{
string S = v.Substring(v.IndexOf("subset") + "subset".Length);
return int.Parse(S.Substring(0, S.IndexOf("_"))) - 3;
}).ToArray();
int[] MapTilts = MapSuffixes.Select(v =>
{
string S = v.Substring(v.IndexOf("tilt") + "tilt".Length);
return int.Parse(S.Substring(0, S.IndexOf(".mrc")));
}).ToArray();
SeriesIndices.Add(Series, SeriesIndices.Count);
float[] MapAngles = MapTilts.Select(t => Series.AnglesCorrect[t]).ToArray();
float[] MapDoses = MapTilts.Select(t => Series.Dose[t]).ToArray();
for (int i = 0; i < MapNames.Length; i++)
{
Image Map = StageDataLoad.LoadMap(Series.WeightOptimizationDir + MapNames[i], new int2(1, 1), 0, typeof(float));
Image MapFT = Map.AsFFT(true);
float[] MapData = MapFT.GetHostContinuousCopy();
Map.Dispose();
MapFT.Dispose();
Image Weights = StageDataLoad.LoadMap(Series.WeightOptimizationDir + WeightNames[i], new int2(1, 1), 0, typeof(float));
float[] WeightsData = Weights.GetHostContinuousCopy();
Weights.Dispose();
Reconstructions.Add(new WeightOptContainer(SeriesIndices[Series], MapSubsets[i], MapData, WeightsData, MapAngles[i], MapDoses[i]));
}
AngleMin = Math.Min(MathHelper.Min(MapAngles), AngleMin);
AngleMax = Math.Max(MathHelper.Max(MapAngles), AngleMax);
DoseMax = Math.Max(MathHelper.Max(MapDoses), DoseMax);
NTilts = Series.NTilts;
//break;
}
float[][] PackedRecFT = new float[SeriesIndices.Count][];
float[][] PackedRecWeights = new float[SeriesIndices.Count][];
foreach (var s in SeriesIndices)
{
WeightOptContainer[] SeriesRecs = Reconstructions.Where(r => r.SeriesID == s.Value).ToArray();
PackedRecFT[s.Value] = new float[SeriesRecs.Length * SeriesRecs[0].DataFT.Length];
PackedRecWeights[s.Value] = new float[SeriesRecs.Length * SeriesRecs[0].DataWeights.Length];
for (int n = 0; n < SeriesRecs.Length; n++)
{
Array.Copy(SeriesRecs[n].DataFT, 0, PackedRecFT[s.Value], n * SeriesRecs[0].DataFT.Length, SeriesRecs[0].DataFT.Length);
Array.Copy(SeriesRecs[n].DataWeights, 0, PackedRecWeights[s.Value], n * SeriesRecs[0].DataWeights.Length, SeriesRecs[0].DataWeights.Length);
}
}
float PixelSize = (float)Options.Movies[0].CTF.PixelSize;
float FreqMin = 1f / (10f / PixelSize), FreqMin2 = FreqMin * FreqMin;
float FreqMax = 1f / (8.5f / PixelSize), FreqMax2 = FreqMax * FreqMax;
int ShellMin = (int)(Dims.X * FreqMin);
int ShellMax = (int)(Dims.X * FreqMax);
int NShells = ShellMax - ShellMin;
float[] R2 = new float[(Dims.X / 2 + 1) * Dims.Y * Dims.Z];
int[] ShellIndices = new int[R2.Length];
for (int z = 0; z < Dims.Z; z++)
{
int zz = z < Dims.Z / 2 + 1 ? z : z - Dims.Z;
zz *= zz;
for (int y = 0; y < Dims.Y; y++)
{
int yy = y < Dims.Y / 2 + 1 ? y : y - Dims.Y;
yy *= yy;
for (int x = 0; x < Dims.X / 2 + 1; x++)
{
int xx = x;
xx *= x;
float r = (float)Math.Sqrt(zz + yy + xx) / Dims.X / PixelSize;
R2[(z * Dims.Y + y) * (Dims.X / 2 + 1) + x] = r * r;
int ir = (int)Math.Round(Math.Sqrt(zz + yy + xx));
ShellIndices[(z * Dims.Y + y) * (Dims.X / 2 + 1) + x] = ir < Dims.X / 2 ? ir : -1;
}
}
}
float[] SeriesWeights = new float[SeriesIndices.Count];
float[] SeriesBfacs = new float[SeriesIndices.Count];
float[] InitGridAngle = new float[NTilts], InitGridDose = new float[NTilts];
for (int i = 0; i < InitGridAngle.Length; i++)
{
InitGridAngle[i] = (float)Math.Cos((i / (float)(InitGridAngle.Length - 1) * (AngleMax - AngleMin) + AngleMin) * Helper.ToRad) * 100f;
InitGridDose[i] = -8 * i / (float)(InitGridAngle.Length - 1) * DoseMax / 10f;
}
CubicGrid GridAngle = new CubicGrid(new int3(NTilts, 1, 1), InitGridAngle);
CubicGrid GridDose = new CubicGrid(new int3(NTilts, 1, 1), InitGridDose);
Func<double[], float[]> WeightedFSC = input =>
{
// Set parameters from input vector
{
int Skip = 0;
GridAngle = new CubicGrid(GridAngle.Dimensions, input.Skip(Skip).Take((int)GridAngle.Dimensions.Elements()).Select(v => (float)v / 100f).ToArray());
Skip += (int)GridAngle.Dimensions.Elements();
GridDose = new CubicGrid(GridDose.Dimensions, input.Skip(Skip).Take((int)GridDose.Dimensions.Elements()).Select(v => (float)v * 10f).ToArray());
}
// Initialize sum vectors
float[] FSC = new float[Dims.X / 2];
float[] MapSum1 = new float[Dims.ElementsFFT() * 2], MapSum2 = new float[Dims.ElementsFFT() * 2];
float[] WeightSum1 = new float[Dims.ElementsFFT()], WeightSum2 = new float[Dims.ElementsFFT()];
int ElementsFT = (int)Dims.ElementsFFT();
foreach (var s in SeriesIndices)
{
WeightOptContainer[] SeriesRecs = Reconstructions.Where(r => r.SeriesID == s.Value).ToArray();
float[] PrecalcWeights = new float[SeriesRecs.Length];
float[] PrecalcBfacs = new float[SeriesRecs.Length];
int[] PrecalcSubsets = new int[SeriesRecs.Length];
for (int n = 0; n < SeriesRecs.Length; n++)
{
WeightOptContainer reconstruction = SeriesRecs[n];
// Weight is Weight(Series) * Weight(Angle) * exp((Bfac(Series) + Bfac(Dose)) / 4 * r^2)
float AngleWeight = GridAngle.GetInterpolated(new float3((reconstruction.Angle - AngleMin) / (AngleMax - AngleMin), 0.5f, 0.5f));
float DoseBfac = GridDose.GetInterpolated(new float3(reconstruction.Dose / DoseMax, 0.5f, 0.5f));
PrecalcWeights[n] = AngleWeight;
PrecalcBfacs[n] = DoseBfac * 0.25f;
PrecalcSubsets[n] = reconstruction.Subset;
}
CPU.OptimizeWeights(SeriesRecs.Length,
PackedRecFT[s.Value],
PackedRecWeights[s.Value],
R2,
ElementsFT,
PrecalcSubsets,
PrecalcBfacs,
PrecalcWeights,
MapSum1,
MapSum2,
WeightSum1,
WeightSum2);
}
for (int i = 0; i < ElementsFT; i++)
{
float Weight = Math.Max(1e-3f, WeightSum1[i]);
MapSum1[i * 2] /= Weight;
MapSum1[i * 2 + 1] /= Weight;
Weight = Math.Max(1e-3f, WeightSum2[i]);
MapSum2[i * 2] /= Weight;
MapSum2[i * 2 + 1] /= Weight;
}
lock (GridAngle)
{
Image Map1FT = new Image(MapSum1, Dims, true, true);
Image Map1 = Map1FT.AsIFFT(true);
Map1.Multiply(SubsetMasks[0]);
Image MaskedFT1 = Map1.AsFFT(true);
float[] MaskedFT1Data = MaskedFT1.GetHostContinuousCopy();
Map1FT.Dispose();
Map1.Dispose();
MaskedFT1.Dispose();
Image Map2FT = new Image(MapSum2, Dims, true, true);
Image Map2 = Map2FT.AsIFFT(true);
Map2.Multiply(SubsetMasks[1]);
Image MaskedFT2 = Map2.AsFFT(true);
float[] MaskedFT2Data = MaskedFT2.GetHostContinuousCopy();
Map2FT.Dispose();
Map2.Dispose();
MaskedFT2.Dispose();
float[] Nums = new float[Dims.X / 2];
float[] Denoms1 = new float[Dims.X / 2];
float[] Denoms2 = new float[Dims.X / 2];
for (int i = 0; i < ElementsFT; i++)
{
int Shell = ShellIndices[i];
if (Shell < 0)
continue;
Nums[Shell] += MaskedFT1Data[i * 2] * MaskedFT2Data[i * 2] + MaskedFT1Data[i * 2 + 1] * MaskedFT2Data[i * 2 + 1];
Denoms1[Shell] += MaskedFT1Data[i * 2] * MaskedFT1Data[i * 2] + MaskedFT1Data[i * 2 + 1] * MaskedFT1Data[i * 2 + 1];
Denoms2[Shell] += MaskedFT2Data[i * 2] * MaskedFT2Data[i * 2] + MaskedFT2Data[i * 2 + 1] * MaskedFT2Data[i * 2 + 1];
}
for (int i = 0; i < Dims.X / 2; i++)
FSC[i] = Nums[i] / (float)Math.Sqrt(Denoms1[i] * Denoms2[i]);
}
return FSC;
};
Func<double[], double> EvalForGrad = input =>
{
return WeightedFSC(input).Skip(ShellMin).Take(NShells).Sum() * Reconstructions.Count;
};
Func<double[], double> Eval = input =>
{
double Score = EvalForGrad(input);
Debug.WriteLine(Score);
return Score;
};
int Iterations = 0;
Func<double[], double[]> Grad = input =>
{
double[] Result = new double[input.Length];
double Step = 1;
if (Iterations++ > 15)
return Result;
//Parallel.For(0, input.Length, new ParallelOptions { MaxDegreeOfParallelism = 4 }, i =>
for (int i = 0; i < input.Length; i++)
{
double[] InputCopy = input.ToList().ToArray();
double Original = InputCopy[i];
InputCopy[i] = Original + Step;
double ResultPlus = EvalForGrad(InputCopy);
InputCopy[i] = Original - Step;
double ResultMinus = EvalForGrad(InputCopy);
InputCopy[i] = Original;
Result[i] = (ResultPlus - ResultMinus) / (Step * 2);
}//);
return Result;
};
List<double> StartParamsList = new List<double>();
StartParamsList.AddRange(GridAngle.FlatValues.Select(v => (double)v));
StartParamsList.AddRange(GridDose.FlatValues.Select(v => (double)v));
double[] StartParams = StartParamsList.ToArray();
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
Optimizer.Epsilon = 3e-7;
Optimizer.Maximize(StartParams);
EvalForGrad(StartParams);
float MaxAngleWeight = MathHelper.Max(GridAngle.FlatValues);
GridAngle = new CubicGrid(GridAngle.Dimensions, GridAngle.FlatValues.Select(v => v / MaxAngleWeight).ToArray());
float MaxDoseBfac = MathHelper.Max(GridDose.FlatValues);
GridDose = new CubicGrid(GridDose.Dimensions, GridDose.FlatValues.Select(v => v - MaxDoseBfac).ToArray());
foreach (var s in Options.Movies)
{
TiltSeries Series = (TiltSeries)s;
List<float> AngleWeights = new List<float>();
List<float> DoseBfacs = new List<float>();
for (int i = 0; i < Series.Angles.Length; i++)
{
float AngleWeight = GridAngle.GetInterpolated(new float3(Math.Min(1, (Series.AnglesCorrect[i] - AngleMin) / (AngleMax - AngleMin)), 0.5f, 0.5f));
float DoseBfac = GridDose.GetInterpolated(new float3(Math.Min(1, Series.Dose[i] / DoseMax), 0.5f, 0.5f));
AngleWeights.Add(AngleWeight);
DoseBfacs.Add(DoseBfac);
}
Series.GridAngleWeights = new CubicGrid(new int3(1, 1, AngleWeights.Count), AngleWeights.ToArray());
Series.GridDoseBfacs = new CubicGrid(new int3(1, 1, DoseBfacs.Count), DoseBfacs.ToArray());
Series.SaveMeta();
}
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="numberOfVariables">The number of free parameters in the optimization problem.</param>
/// <param name="constraints">The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangianSolver(int numberOfVariables,
IEnumerable<NonlinearConstraint> constraints)
{
var innerSolver = new BroydenFletcherGoldfarbShanno(numberOfVariables);
init(numberOfVariables, constraints, innerSolver);
}
/// <summary>
/// Creates a new instance of the Augmented Lagrangian algorithm.
/// </summary>
///
/// <param name="numberOfVariables">The number of free parameters in the optimization problem.</param>
/// <param name="constraints">The <see cref="NonlinearConstraint"/>s to which the solution must be subjected.</param>
///
public AugmentedLagrangianSolver(int numberOfVariables, IEnumerable<NonlinearConstraint> constraints)
{
this.numberOfVariables = numberOfVariables;
List<NonlinearConstraint> equality = new List<NonlinearConstraint>();
List<NonlinearConstraint> lesserThan = new List<NonlinearConstraint>();
List<NonlinearConstraint> greaterThan = new List<NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.Solution = new double[numberOfVariables];
dualSolver = new BroydenFletcherGoldfarbShanno(numberOfVariables);
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
//for (int i = 0; i < Solution.Length; i++)
// Solution[i] = Accord.Math.Tools.Random.NextDouble() * 2 - 1;
}
private void init(NonlinearObjectiveFunction function,
IEnumerable<NonlinearConstraint> constraints, IGradientOptimizationMethod innerSolver)
{
if (function != null)
{
if (function.NumberOfVariables != NumberOfVariables)
{
throw new ArgumentOutOfRangeException("function",
"Incorrect number of variables in the objective function. " +
"The number of variables must match the number of variables set in the solver.");
}
this.Function = function.Function;
this.Gradient = function.Gradient;
}
if (innerSolver == null)
{
innerSolver = new BroydenFletcherGoldfarbShanno(NumberOfVariables)
{
LineSearch = Optimization.LineSearch.BacktrackingArmijo,
Corrections = 10
};
}
List<NonlinearConstraint> equality = new List<NonlinearConstraint>();
List<NonlinearConstraint> lesserThan = new List<NonlinearConstraint>();
List<NonlinearConstraint> greaterThan = new List<NonlinearConstraint>();
foreach (var c in constraints)
{
switch (c.ShouldBe)
{
case ConstraintType.EqualTo:
equality.Add(c); break;
case ConstraintType.GreaterThanOrEqualTo:
greaterThan.Add(c); break;
case ConstraintType.LesserThanOrEqualTo:
lesserThan.Add(c); break;
default:
throw new ArgumentException("Unknown constraint type.", "constraints");
}
}
this.lesserThanConstraints = lesserThan.ToArray();
this.greaterThanConstraints = greaterThan.ToArray();
this.equalityConstraints = equality.ToArray();
this.lambda = new double[equalityConstraints.Length];
this.mu = new double[lesserThanConstraints.Length];
this.nu = new double[greaterThanConstraints.Length];
this.dualSolver = innerSolver;
dualSolver.Function = objectiveFunction;
dualSolver.Gradient = objectiveGradient;
}
public void ProcessParticleShift(MapHeader originalHeader, Image originalStack, Star stardata, Image refft, Image maskft, int dimbox, decimal scaleFactor)
{
// Deal with dimensions and grids.
int NFrames = originalHeader.Dimensions.Z;
int2 DimsImage = new int2(originalHeader.Dimensions);
int2 DimsRegion = new int2(dimbox, dimbox);
decimal SubdivisionRatio = 4M;
List<int3> PyramidSizes = new List<int3>();
PyramidSizes.Add(new int3(MainWindow.Options.GridMoveX, MainWindow.Options.GridMoveX, Math.Min(NFrames, MainWindow.Options.GridMoveZ)));
while (true)
{
int3 Previous = PyramidSizes.Last();
int NewZ = Math.Min((int)Math.Round(Previous.Z / SubdivisionRatio), Previous.Z - 1);
if (NewZ < 2)
break;
PyramidSizes.Add(new int3(Previous.X * 2, Previous.Y * 2, NewZ));
}
PyramidShiftX.Clear();
PyramidShiftY.Clear();
float3[] PositionsGrid, PositionsGridPerFrame;
float2[] PositionsExtraction, PositionsShift;
float3[] ParticleAngles;
List<int> RowIndices = new List<int>();
{
string[] ColumnNames = stardata.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnNames.Length; i++)
if (ColumnNames[i].Contains(RootName))
RowIndices.Add(i);
string[] ColumnOriginX = stardata.GetColumn("rlnCoordinateX");
string[] ColumnOriginY = stardata.GetColumn("rlnCoordinateY");
string[] ColumnShiftX = stardata.GetColumn("rlnOriginX");
string[] ColumnShiftY = stardata.GetColumn("rlnOriginY");
string[] ColumnAngleRot = stardata.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = stardata.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = stardata.GetColumn("rlnAnglePsi");
PositionsGrid = new float3[RowIndices.Count];
PositionsGridPerFrame = new float3[RowIndices.Count * NFrames];
PositionsExtraction = new float2[RowIndices.Count];
PositionsShift = new float2[RowIndices.Count * NFrames];
ParticleAngles = new float3[RowIndices.Count];
{
int i = 0;
foreach (var nameIndex in RowIndices)
{
float OriginX = float.Parse(ColumnOriginX[nameIndex]);
float OriginY = float.Parse(ColumnOriginY[nameIndex]);
float ShiftX = float.Parse(ColumnShiftX[nameIndex]);
float ShiftY = float.Parse(ColumnShiftY[nameIndex]);
PositionsExtraction[i] = new float2(OriginX - ShiftX, OriginY - ShiftY);
PositionsGrid[i] = new float3((OriginX - ShiftX) / DimsImage.X, (OriginY - ShiftY) / DimsImage.Y, 0.5f);
for (int z = 0; z < NFrames; z++)
{
PositionsGridPerFrame[z * RowIndices.Count + i] = new float3(PositionsGrid[i].X,
PositionsGrid[i].Y,
(float)z / (NFrames - 1));
PositionsShift[z * RowIndices.Count + i] = GetShiftFromPyramid(PositionsGridPerFrame[z * RowIndices.Count + i]);
}
ParticleAngles[i] = new float3(float.Parse(ColumnAngleRot[nameIndex]) * Helper.ToRad,
float.Parse(ColumnAngleTilt[nameIndex]) * Helper.ToRad,
float.Parse(ColumnAnglePsi[nameIndex]) * Helper.ToRad);
i++;
}
}
}
int NPositions = PositionsGrid.Length;
if (NPositions == 0)
return;
int MinFreqInclusive = (int)(MainWindow.Options.MovementRangeMin * DimsRegion.X / 2);
int MaxFreqExclusive = (int)(MainWindow.Options.MovementRangeMax * DimsRegion.X / 2);
int NFreq = MaxFreqExclusive - MinFreqInclusive;
int CentralFrame = NFrames / 2;
int MaskExpansions = 4; // Math.Max(1, PyramidSizes[0].Z / 3);
int[] MaskSizes = new int[MaskExpansions];
// Allocate memory and create all prerequisites:
int MaskLength;
Image ShiftFactors;
Image Phases;
Image Projections;
Image Shifts;
Image InvSigma;
{
List<long> Positions = new List<long>();
List<float2> Factors = new List<float2>();
List<float2> Freq = new List<float2>();
int Min2 = MinFreqInclusive * MinFreqInclusive;
int Max2 = MaxFreqExclusive * MaxFreqExclusive;
for (int y = 0; y < DimsRegion.Y; y++)
{
int yy = y - DimsRegion.X / 2;
for (int x = 0; x < DimsRegion.X / 2 + 1; x++)
{
int xx = x - DimsRegion.X / 2;
int r2 = xx * xx + yy * yy;
if (r2 >= Min2 && r2 < Max2)
{
Positions.Add(y * (DimsRegion.X / 2 + 1) + x);
Factors.Add(new float2((float)xx / DimsRegion.X * 2f * (float)Math.PI,
(float)yy / DimsRegion.X * 2f * (float)Math.PI));
float Angle = (float)Math.Atan2(yy, xx);
float r = (float)Math.Sqrt(r2);
Freq.Add(new float2(r, Angle));
}
}
}
// Addresses for CTF simulation
Image CTFCoordsCart = new Image(new int3(DimsRegion), true, true);
{
float2[] CoordsData = new float2[CTFCoordsCart.ElementsSliceComplex];
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) => CoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r / DimsRegion.X, a));
CTFCoordsCart.UpdateHostWithComplex(new[] { CoordsData });
CTFCoordsCart.RemapToFT();
}
float[] ValuesDefocus = GridCTF.GetInterpolatedNative(PositionsGrid);
CTFStruct[] PositionsCTF = ValuesDefocus.Select(v =>
{
CTF Altered = CTF.GetCopy();
Altered.PixelSizeDelta = 0;
Altered.Defocus = (decimal)v;
//Altered.Bfactor = -MainWindow.Options.MovementBfactor;
return Altered.ToStruct();
}).ToArray();
// Sort everyone with ascending distance from center.
List<KeyValuePair<float, int>> FreqIndices = Freq.Select((v, i) => new KeyValuePair<float, int>(v.X, i)).ToList();
FreqIndices.Sort((a, b) => a.Key.CompareTo(b.Key));
int[] SortedIndices = FreqIndices.Select(v => v.Value).ToArray();
Helper.Reorder(Positions, SortedIndices);
Helper.Reorder(Factors, SortedIndices);
Helper.Reorder(Freq, SortedIndices);
long[] RelevantMask = Positions.ToArray();
ShiftFactors = new Image(Helper.ToInterleaved(Factors.ToArray()));
MaskLength = RelevantMask.Length;
// Get mask sizes for different expansion steps.
for (int i = 0; i < MaskExpansions; i++)
{
float CurrentMaxFreq = MinFreqInclusive + (MaxFreqExclusive - MinFreqInclusive) / (float)MaskExpansions * (i + 1);
MaskSizes[i] = Freq.Count(v => v.X * v.X < CurrentMaxFreq * CurrentMaxFreq);
}
Phases = new Image(IntPtr.Zero, new int3(MaskLength, NPositions, NFrames), false, true, false);
Projections = new Image(IntPtr.Zero, new int3(MaskLength, NPositions, NFrames), false, true, false);
InvSigma = new Image(IntPtr.Zero, new int3(MaskLength, 1, 1));
Image ParticleMasksFT = maskft.AsProjections(ParticleAngles, DimsRegion, MainWindow.Options.ProjectionOversample);
Image ParticleMasks = ParticleMasksFT.AsIFFT();
ParticleMasksFT.Dispose();
ParticleMasks.RemapFromFT();
Parallel.ForEach(ParticleMasks.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(25f, slice[i])) - 2) / 23f;
});
Image ProjectionsSparse = refft.AsProjections(ParticleAngles, DimsRegion, MainWindow.Options.ProjectionOversample);
Image InvSigmaSparse = new Image(new int3(DimsRegion), true);
{
int GroupNumber = int.Parse(stardata.GetRowValue(RowIndices[0], "rlnGroupNumber"));
//Star SigmaTable = new Star("D:\\rado27\\RefineWarppolish\\run1_model.star", "data_model_group_" + GroupNumber);
Star SigmaTable = new Star(MainWindow.Options.ModelStarPath, "data_model_group_" + GroupNumber);
float[] SigmaValues = SigmaTable.GetColumn("rlnSigma2Noise").Select(v => float.Parse(v)).ToArray();
float[] Sigma2NoiseData = InvSigmaSparse.GetHost(Intent.Write)[0];
Helper.ForEachElementFT(new int2(DimsRegion.X, DimsRegion.Y), (x, y, xx, yy, r, angle) =>
{
int ir = (int)r;
float val = 0;
if (ir < SigmaValues.Length)
{
if (SigmaValues[ir] != 0f)
val = 1f / SigmaValues[ir];
}
Sigma2NoiseData[y * (DimsRegion.X / 2 + 1) + x] = val;
});
float MaxSigma = MathHelper.Max(Sigma2NoiseData);
for (int i = 0; i < Sigma2NoiseData.Length; i++)
Sigma2NoiseData[i] /= MaxSigma;
InvSigmaSparse.RemapToFT();
}
//InvSigmaSparse.WriteMRC("d_sigma2noise.mrc");
float PixelSize = (float)CTF.PixelSize;
float PixelDelta = (float)CTF.PixelSizeDelta;
float PixelAngle = (float)CTF.PixelSizeAngle * Helper.ToRad;
GPU.CreateParticleShift(originalStack.GetDevice(Intent.Read),
DimsImage,
NFrames,
Helper.ToInterleaved(PositionsExtraction),
Helper.ToInterleaved(PositionsShift),
NPositions,
DimsRegion,
RelevantMask,
(uint)RelevantMask.Length,
ParticleMasks.GetDevice(Intent.Read),
ProjectionsSparse.GetDevice(Intent.Read),
PositionsCTF,
CTFCoordsCart.GetDevice(Intent.Read),
InvSigmaSparse.GetDevice(Intent.Read),
PixelSize + PixelDelta / 2,
PixelSize - PixelDelta / 2,
PixelAngle,
Phases.GetDevice(Intent.Write),
Projections.GetDevice(Intent.Write),
InvSigma.GetDevice(Intent.Write));
InvSigmaSparse.Dispose();
ParticleMasks.Dispose();
ProjectionsSparse.Dispose();
CTFCoordsCart.Dispose();
originalStack.FreeDevice();
Shifts = new Image(new float[NPositions * NFrames * 2]);
}
#region Fit movement
{
int NPyramidPoints = 0;
float[][][] WiggleWeights = new float[PyramidSizes.Count][][];
for (int p = 0; p < PyramidSizes.Count; p++)
{
CubicGrid WiggleGrid = new CubicGrid(PyramidSizes[p]);
NPyramidPoints += (int)PyramidSizes[p].Elements();
WiggleWeights[p] = WiggleGrid.GetWiggleWeights(PositionsGridPerFrame);
}
double[] StartParams = new double[NPyramidPoints * 2];
for (int m = 3; m < MaskExpansions; m++)
{
for (int currentGrid = 0; currentGrid < PyramidSizes.Count; currentGrid++)
{
Action<double[]> SetPositions = input =>
{
// Construct CubicGrids and get interpolated shift values.
float[] AlteredX = new float[PositionsGridPerFrame.Length];
float[] AlteredY = new float[PositionsGridPerFrame.Length];
int Offset = 0;
foreach (var size in PyramidSizes)
{
int Elements = (int)size.Elements();
CubicGrid GridX = new CubicGrid(size, input.Skip(Offset).Take(Elements).Select(v => (float)v).ToArray());
AlteredX = MathHelper.Plus(AlteredX, GridX.GetInterpolatedNative(PositionsGridPerFrame));
CubicGrid GridY = new CubicGrid(size, input.Skip(NPyramidPoints + Offset).Take(Elements).Select(v => (float)v).ToArray());
AlteredY = MathHelper.Plus(AlteredY, GridY.GetInterpolatedNative(PositionsGridPerFrame));
Offset += Elements;
}
// Finally, set the shift values in the device array.
float[] ShiftData = Shifts.GetHost(Intent.Write)[0];
for (int i = 0; i < PositionsGridPerFrame.Length; i++)
{
ShiftData[i * 2] = AlteredX[i];
ShiftData[i * 2 + 1] = AlteredY[i];
}
};
Func<double[], double> Eval = input =>
{
SetPositions(input);
float[] Diff = new float[NPositions * NFrames];
GPU.ParticleShiftGetDiff(Phases.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
InvSigma.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
Diff,
(uint)NPositions,
(uint)NFrames);
//for (int i = 0; i < Diff.Length; i++)
//Diff[i] = Diff[i] * 100f;
double Score = Diff.Sum();
//Debug.WriteLine(Score);
return Score;
};
Func<double[], double[]> Grad = input =>
{
SetPositions(input);
float[] Diff = new float[NPositions * NFrames * 2];
GPU.ParticleShiftGetGrad(Phases.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
InvSigma.GetDevice(Intent.Read),
(uint)MaskLength,
(uint)MaskSizes[m],
Shifts.GetDevice(Intent.Read),
Diff,
(uint)NPositions,
(uint)NFrames);
//for (int i = 0; i < Diff.Length; i++)
//Diff[i] = Diff[i] * 100f;
float[] DiffX = new float[NPositions * NFrames], DiffY = new float[NPositions * NFrames];
for (int i = 0; i < DiffX.Length; i++)
{
DiffX[i] = Diff[i * 2];
DiffY[i] = Diff[i * 2 + 1];
}
double[] Result = new double[input.Length];
int Offset = 0;
for (int p = 0; p < PyramidSizes.Count; p++)
{
//if (p == currentGrid)
Parallel.For(0, (int)PyramidSizes[p].Elements(), i =>
{
Result[Offset + i] = MathHelper.ReduceWeighted(DiffX, WiggleWeights[p][i]);
Result[NPyramidPoints + Offset + i] = MathHelper.ReduceWeighted(DiffY, WiggleWeights[p][i]);
});
Offset += (int)PyramidSizes[p].Elements();
}
return Result;
};
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
//Optimizer.Corrections = 20;
Optimizer.Minimize(StartParams);
}
{
PyramidShiftX.Clear();
PyramidShiftY.Clear();
int Offset = 0;
foreach (var size in PyramidSizes)
{
int Elements = (int)size.Elements();
CubicGrid GridX = new CubicGrid(size, StartParams.Skip(Offset).Take(Elements).Select(v => (float)v).ToArray());
PyramidShiftX.Add(GridX);
CubicGrid GridY = new CubicGrid(size, StartParams.Skip(NPyramidPoints + Offset).Take(Elements).Select(v => (float)v).ToArray());
PyramidShiftY.Add(GridY);
Offset += Elements;
}
}
}
}
#endregion
ShiftFactors.Dispose();
Phases.Dispose();
Projections.Dispose();
Shifts.Dispose();
InvSigma.Dispose();
SaveMeta();
}
public void lbfgsTest2()
{
// Suppose we would like to find the minimum of the function
//
// f(x,y) = -exp{-(x-1)²} - exp{-(y-2)²/2}
//
// First we need write down the function either as a named
// method, an anonymous method or as a lambda function:
Func<double[], double> f = (x) =>
-Math.Exp(-Math.Pow(x[0] - 1, 2)) - Math.Exp(-0.5 * Math.Pow(x[1] - 2, 2));
// Now, we need to write its gradient, which is just the
// vector of first partial derivatives del_f / del_x, as:
//
// g(x,y) = { del f / del x, del f / del y }
//
Func<double[], double[]> g = (x) => new double[]
{
// df/dx = {-2 e^(- (x-1)^2) (x-1)}
2 * Math.Exp(-Math.Pow(x[0] - 1, 2)) * (x[0] - 1),
// df/dy = {- e^(-1/2 (y-2)^2) (y-2)}
Math.Exp(-0.5 * Math.Pow(x[1] - 2, 2)) * (x[1] - 2)
};
// Finally, we can create the L-BFGS solver, passing the functions as arguments
var lbfgs = new BroydenFletcherGoldfarbShanno(numberOfVariables: 2, function: f, gradient: g);
// And then minimize the function:
double minValue = lbfgs.Minimize();
double[] solution = lbfgs.Solution;
// The resultant minimum value should be -2, and the solution
// vector should be { 1.0, 2.0 }. The answer can be checked on
// Wolfram Alpha by clicking the following the link:
// http://www.wolframalpha.com/input/?i=maximize+%28exp%28-%28x-1%29%C2%B2%29+%2B+exp%28-%28y-2%29%C2%B2%2F2%29%29
double expected = -2;
Assert.AreEqual(expected, minValue, 1e-10);
Assert.AreEqual(1, solution[0], 1e-6);
Assert.AreEqual(2, solution[1], 1e-6);
}
public void ConstructorTest3()
{
// minimize f(x) = x*y*z,
// s.t.
//
// 1 - x² - 2y² - 3z² > 0
// x > 0,
// y > 0
//
// Easy three dimensional minimization in ellipsoid.
var function = new NonlinearObjectiveFunction(3,
function: x => x[0] * x[1] * x[2],
gradient: x => new[] { x[1] * x[2], x[0] * x[2], x[0] * x[1] });
NonlinearConstraint[] constraints =
{
new NonlinearConstraint(3,
function: x => 1.0 - x[0] * x[0] - 2.0 * x[1] * x[1] - 3.0 * x[2] * x[2],
gradient: x => new[] { -2.0 * x[0], -4.0 * x[1], -6.0 * x[2] }),
new NonlinearConstraint(3,
function: x => x[0],
gradient: x => new[] { 1.0, 0, 0 }),
new NonlinearConstraint(3,
function: x => x[1],
gradient: x => new[] { 0, 1.0, 0 }),
new NonlinearConstraint(3,
function: x => -x[2],
gradient: x => new[] { 0, 0, -1.0 }),
};
for (int i = 0; i < constraints.Length; i++)
{
Assert.AreEqual(ConstraintType.GreaterThanOrEqualTo, constraints[i].ShouldBe);
Assert.AreEqual(0, constraints[i].Value);
}
var inner = new BroydenFletcherGoldfarbShanno(3);
inner.LineSearch = LineSearch.BacktrackingArmijo;
inner.Corrections = 10;
var solver = new AugmentedLagrangian(inner, function, constraints);
Assert.AreEqual(inner, solver.Optimizer);
Assert.IsTrue(solver.Minimize());
double minimum = solver.Value;
double[] solution = solver.Solution;
double[] expected =
{
1.0 / Math.Sqrt(3.0), 1.0 / Math.Sqrt(6.0), -1.0 / 3.0
};
for (int i = 0; i < expected.Length; i++)
Assert.AreEqual(expected[i], solver.Solution[i], 1e-3);
Assert.AreEqual(-0.078567420132031968, minimum, 1e-4);
double expectedMinimum = function.Function(solver.Solution);
Assert.AreEqual(expectedMinimum, minimum);
}
public OptimizationProgressEventArgs[] Actual(Specification problem)
{
BroydenFletcherGoldfarbShanno target = new BroydenFletcherGoldfarbShanno(problem.Variables)
{
Corrections = m,
Epsilon = epsilon,
Past = past,
Delta = delta,
MaxIterations = max_iterations,
LineSearch = (LineSearch)linesearch,
MaxLineSearch = max_linesearch,
MinStep = min_step,
MaxStep = max_step,
ParameterTolerance = ftol,
Wolfe = wolfe,
GradientTolerance = gtol,
FunctionTolerance = xtol,
OrthantwiseC = orthantwise_c,
OrthantwiseStart = orthantwise_start,
OrthantwiseEnd = orthantwise_end
};
target.Function = problem.Function;
target.Gradient = problem.Gradient;
actual.Clear();
target.Progress += new EventHandler<OptimizationProgressEventArgs>(target_Progress);
target.Minimize((double[])problem.Start.Clone());
ActualMessage = target.Status.GetDescription();
return actual.ToArray();
}
public void NoFunctionTest()
{
BroydenFletcherGoldfarbShanno target = new BroydenFletcherGoldfarbShanno(2);
target.Minimize();
}
private void OptimizePerTomoWeights()
{
if (!Options.Movies.Any(m => m.GetType() == typeof (TiltSeries)))
return;
Image Mask1 = StageDataLoad.LoadMap("F:\\chloroplastribo\\vlion\\mask_post.mrc", new int2(1, 1), 0, typeof (float));
//Image Mask2 = StageDataLoad.LoadMap("F:\\badaben\\vlion\\mask_C3_post.mrc", new int2(1, 1), 0, typeof(float));
//Image Mask3 = StageDataLoad.LoadMap("F:\\badaben\\vlion\\mask_C4_post.mrc", new int2(1, 1), 0, typeof(float));
List<Image> SubsetMasks = new List<Image> { Mask1, Mask1 };
int3 Dims = Mask1.Dims;
List<WeightOptContainer> Reconstructions = new List<WeightOptContainer>();
Dictionary<TiltSeries, int> SeriesIndices = new Dictionary<TiltSeries, int>();
foreach (Movie movie in Options.Movies)
{
if (!movie.DoProcess)
continue;
TiltSeries Series = (TiltSeries)movie;
string[] FileNames = Directory.EnumerateFiles(Series.WeightOptimizationDir, Series.RootName + "_subset*.mrc").Where(v => v.Contains("subset1") || v.Contains("subset2")).Select(v => new FileInfo(v).Name).ToArray();
if (FileNames.Length == 0)
continue;
string[] MapNames = FileNames.Where(v => !v.Contains(".weight.mrc")).ToArray();
string[] WeightNames = FileNames.Where(v => v.Contains(".weight.mrc")).ToArray();
if (MapNames.Length != WeightNames.Length)
throw new Exception("Number of reconstructions and weights does not match!");
string[] MapSuffixes = MapNames.Select(v => v.Substring(Series.RootName.Length)).ToArray();
int[] MapSubsets = MapSuffixes.Select(v =>
{
string S = v.Substring(v.IndexOf("subset") + "subset".Length);
return int.Parse(S.Substring(0, S.IndexOf(".mrc"))) - 1;
}).ToArray();
SeriesIndices.Add(Series, SeriesIndices.Count);
for (int i = 0; i < MapNames.Length; i++)
{
Image Map = StageDataLoad.LoadMap(Series.WeightOptimizationDir + MapNames[i], new int2(1, 1), 0, typeof (float));
Image MapFT = Map.AsFFT(true);
float[] MapData = MapFT.GetHostContinuousCopy();
Map.Dispose();
MapFT.Dispose();
Image Weights = StageDataLoad.LoadMap(Series.WeightOptimizationDir + WeightNames[i], new int2(1, 1), 0, typeof (float));
float[] WeightsData = Weights.GetHostContinuousCopy();
Weights.Dispose();
Reconstructions.Add(new WeightOptContainer(SeriesIndices[Series], MapSubsets[i], MapData, WeightsData, 0, 0));
}
//break;
}
float[][] PackedRecFT = new float[SeriesIndices.Count][];
float[][] PackedRecWeights = new float[SeriesIndices.Count][];
foreach (var s in SeriesIndices)
{
WeightOptContainer[] SeriesRecs = Reconstructions.Where(r => r.SeriesID == s.Value).ToArray();
PackedRecFT[s.Value] = new float[SeriesRecs.Length * SeriesRecs[0].DataFT.Length];
PackedRecWeights[s.Value] = new float[SeriesRecs.Length * SeriesRecs[0].DataWeights.Length];
for (int n = 0; n < SeriesRecs.Length; n++)
{
Array.Copy(SeriesRecs[n].DataFT, 0, PackedRecFT[s.Value], n * SeriesRecs[0].DataFT.Length, SeriesRecs[0].DataFT.Length);
Array.Copy(SeriesRecs[n].DataWeights, 0, PackedRecWeights[s.Value], n * SeriesRecs[0].DataWeights.Length, SeriesRecs[0].DataWeights.Length);
}
}
float PixelSize = (float)Options.Movies[0].CTF.PixelSize;
float FreqMin = 1f / (18.0f / PixelSize), FreqMin2 = FreqMin * FreqMin;
float FreqMax = 1f / (14.5f / PixelSize), FreqMax2 = FreqMax * FreqMax;
int ShellMin = (int)(Dims.X * FreqMin);
int ShellMax = (int)(Dims.X * FreqMax);
int NShells = ShellMax - ShellMin;
float[] R2 = new float[(Dims.X / 2 + 1) * Dims.Y * Dims.Z];
int[] ShellIndices = new int[R2.Length];
for (int z = 0; z < Dims.Z; z++)
{
int zz = z < Dims.Z / 2 + 1 ? z : z - Dims.Z;
zz *= zz;
for (int y = 0; y < Dims.Y; y++)
{
int yy = y < Dims.Y / 2 + 1 ? y : y - Dims.Y;
yy *= yy;
for (int x = 0; x < Dims.X / 2 + 1; x++)
{
int xx = x;
xx *= x;
float r = (float)Math.Sqrt(zz + yy + xx) / Dims.X / PixelSize;
R2[(z * Dims.Y + y) * (Dims.X / 2 + 1) + x] = r * r;
int ir = (int)Math.Round(Math.Sqrt(zz + yy + xx));
ShellIndices[(z * Dims.Y + y) * (Dims.X / 2 + 1) + x] = ir < Dims.X / 2 ? ir : -1;
}
}
}
float[] SeriesWeights = new float[SeriesIndices.Count];
float[] SeriesBfacs = new float[SeriesIndices.Count];
Func<double[], float[]> WeightedFSC = input =>
{
// Set parameters from input vector
//{
int Skip = 0;
SeriesWeights = input.Take(SeriesWeights.Length).Select(v => (float)v / 100f).ToArray();
Skip += SeriesWeights.Length;
SeriesBfacs = input.Skip(Skip).Take(SeriesBfacs.Length).Select(v => (float)v * 10f).ToArray();
//}
// Initialize sum vectors
float[] FSC = new float[Dims.X / 2];
float[] MapSum1 = new float[Dims.ElementsFFT() * 2], MapSum2 = new float[Dims.ElementsFFT() * 2];
float[] WeightSum1 = new float[Dims.ElementsFFT()], WeightSum2 = new float[Dims.ElementsFFT()];
int ElementsFT = (int)Dims.ElementsFFT();
foreach (var s in SeriesIndices)
{
WeightOptContainer[] SeriesRecs = Reconstructions.Where(r => r.SeriesID == s.Value).ToArray();
float[] PrecalcWeights = new float[SeriesRecs.Length];
float[] PrecalcBfacs = new float[SeriesRecs.Length];
int[] PrecalcSubsets = new int[SeriesRecs.Length];
for (int n = 0; n < SeriesRecs.Length; n++)
{
WeightOptContainer reconstruction = SeriesRecs[n];
// Weight is Weight(Series) * exp(Bfac(Series) / 4 * r^2)
float SeriesWeight = (float)Math.Exp(SeriesWeights[reconstruction.SeriesID]);
float SeriesBfac = SeriesBfacs[reconstruction.SeriesID];
PrecalcWeights[n] = SeriesWeight;
PrecalcBfacs[n] = SeriesBfac * 0.25f;
PrecalcSubsets[n] = reconstruction.Subset;
}
CPU.OptimizeWeights(SeriesRecs.Length,
PackedRecFT[s.Value],
PackedRecWeights[s.Value],
R2,
ElementsFT,
PrecalcSubsets,
PrecalcBfacs,
PrecalcWeights,
MapSum1,
MapSum2,
WeightSum1,
WeightSum2);
}
for (int i = 0; i < ElementsFT; i++)
{
float Weight = Math.Max(1e-3f, WeightSum1[i]);
MapSum1[i * 2] /= Weight;
MapSum1[i * 2 + 1] /= Weight;
Weight = Math.Max(1e-3f, WeightSum2[i]);
MapSum2[i * 2] /= Weight;
MapSum2[i * 2 + 1] /= Weight;
}
Image Map1FT = new Image(MapSum1, Dims, true, true);
Image Map1 = Map1FT.AsIFFT(true);
Map1.Multiply(SubsetMasks[0]);
Image MaskedFT1 = Map1.AsFFT(true);
float[] MaskedFT1Data = MaskedFT1.GetHostContinuousCopy();
Map1FT.Dispose();
Map1.Dispose();
MaskedFT1.Dispose();
Image Map2FT = new Image(MapSum2, Dims, true, true);
Image Map2 = Map2FT.AsIFFT(true);
Map2.Multiply(SubsetMasks[1]);
Image MaskedFT2 = Map2.AsFFT(true);
float[] MaskedFT2Data = MaskedFT2.GetHostContinuousCopy();
Map2FT.Dispose();
Map2.Dispose();
MaskedFT2.Dispose();
float[] Nums = new float[Dims.X / 2];
float[] Denoms1 = new float[Dims.X / 2];
float[] Denoms2 = new float[Dims.X / 2];
for (int i = 0; i < ElementsFT; i++)
{
int Shell = ShellIndices[i];
if (Shell < 0)
continue;
Nums[Shell] += MaskedFT1Data[i * 2] * MaskedFT2Data[i * 2] + MaskedFT1Data[i * 2 + 1] * MaskedFT2Data[i * 2 + 1];
Denoms1[Shell] += MaskedFT1Data[i * 2] * MaskedFT1Data[i * 2] + MaskedFT1Data[i * 2 + 1] * MaskedFT1Data[i * 2 + 1];
Denoms2[Shell] += MaskedFT2Data[i * 2] * MaskedFT2Data[i * 2] + MaskedFT2Data[i * 2 + 1] * MaskedFT2Data[i * 2 + 1];
}
for (int i = 0; i < Dims.X / 2; i++)
FSC[i] = Nums[i] / (float)Math.Sqrt(Denoms1[i] * Denoms2[i]);
return FSC;
};
Func<double[], double> EvalForGrad = input =>
{
return WeightedFSC(input).Skip(ShellMin).Take(NShells).Sum() * Reconstructions.Count;
};
Func<double[], double> Eval = input =>
{
double Score = EvalForGrad(input);
Debug.WriteLine(Score);
return Score;
};
int Iterations = 0;
Func<double[], double[]> Grad = input =>
{
double[] Result = new double[input.Length];
double Step = 4;
if (Iterations++ > 15)
return Result;
//Parallel.For(0, input.Length, new ParallelOptions { MaxDegreeOfParallelism = 4 }, i =>
for (int i = 0; i < input.Length; i++)
{
double[] InputCopy = input.ToList().ToArray();
double Original = InputCopy[i];
InputCopy[i] = Original + Step;
double ResultPlus = EvalForGrad(InputCopy);
InputCopy[i] = Original - Step;
double ResultMinus = EvalForGrad(InputCopy);
InputCopy[i] = Original;
Result[i] = (ResultPlus - ResultMinus) / (Step * 2);
}//);
return Result;
};
List<double> StartParamsList = new List<double>();
StartParamsList.AddRange(SeriesWeights.Select(v => (double)v));
StartParamsList.AddRange(SeriesBfacs.Select(v => (double)v));
double[] StartParams = StartParamsList.ToArray();
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
Optimizer.Epsilon = 3e-7;
Optimizer.Maximize(StartParams);
EvalForGrad(StartParams);
foreach (var s in SeriesIndices)
{
s.Key.GlobalWeight = (float)Math.Exp(SeriesWeights[s.Value] - MathHelper.Max(SeriesWeights)); // Minus, because exponential
s.Key.GlobalBfactor = SeriesBfacs[s.Value] - MathHelper.Max(SeriesBfacs);
s.Key.SaveMeta();
}
}
public void ProcessParticleCTF(MapHeader originalHeader, Image originalStack, Star stardata, Image refft, Image maskft, int dimbox, decimal scaleFactor)
{
//CTF.Cs = MainWindow.Options.CTFCs;
#region Dimensions and grids
int NFrames = originalHeader.Dimensions.Z;
int2 DimsImage = new int2(originalHeader.Dimensions);
int2 DimsRegion = new int2(dimbox, dimbox);
float3[] PositionsGrid;
float3[] PositionsExtraction;
float3[] ParticleAngles;
List<int> RowIndices = new List<int>();
{
string[] ColumnNames = stardata.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnNames.Length; i++)
if (ColumnNames[i].Contains(RootName))
RowIndices.Add(i);
string[] ColumnOriginX = stardata.GetColumn("rlnCoordinateX");
string[] ColumnOriginY = stardata.GetColumn("rlnCoordinateY");
string[] ColumnShiftX = stardata.GetColumn("rlnOriginX");
string[] ColumnShiftY = stardata.GetColumn("rlnOriginY");
string[] ColumnAngleRot = stardata.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = stardata.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = stardata.GetColumn("rlnAnglePsi");
PositionsGrid = new float3[RowIndices.Count];
PositionsExtraction = new float3[RowIndices.Count];
ParticleAngles = new float3[RowIndices.Count];
{
int i = 0;
foreach (var nameIndex in RowIndices)
{
float OriginX = float.Parse(ColumnOriginX[nameIndex]);
float OriginY = float.Parse(ColumnOriginY[nameIndex]);
float ShiftX = float.Parse(ColumnShiftX[nameIndex]);
float ShiftY = float.Parse(ColumnShiftY[nameIndex]);
PositionsExtraction[i] = new float3(OriginX - ShiftX - dimbox / 2, OriginY - ShiftY - dimbox / 2, 0f);
PositionsGrid[i] = new float3((OriginX - ShiftX) / DimsImage.X, (OriginY - ShiftY) / DimsImage.Y, 0);
ParticleAngles[i] = new float3(float.Parse(ColumnAngleRot[nameIndex]) * Helper.ToRad,
float.Parse(ColumnAngleTilt[nameIndex]) * Helper.ToRad,
float.Parse(ColumnAnglePsi[nameIndex]) * Helper.ToRad);
i++;
}
}
}
int NPositions = PositionsGrid.Length;
if (NPositions == 0)
return;
int CTFGridX = MainWindow.Options.GridCTFX;
int CTFGridY = MainWindow.Options.GridCTFY;
int CTFGridZ = Math.Min(NFrames, MainWindow.Options.GridCTFZ);
int FrameGroupSize = CTFGridZ > 1 ? 12 : 1;
int NFrameGroups = CTFGridZ > 1 ? NFrames / FrameGroupSize : 1;
GridCTF = GridCTF.Resize(new int3(CTFGridX, CTFGridY, CTFGridZ));
GridCTFPhase = GridCTFPhase.Resize(new int3(1, 1, CTFGridZ));
int NSpectra = NFrameGroups * NPositions;
int MinFreqInclusive = (int)(MainWindow.Options.CTFRangeMin * DimsRegion.X / 2);
int MaxFreqExclusive = (int)(MainWindow.Options.CTFRangeMax * DimsRegion.X / 2);
int NFreq = MaxFreqExclusive - MinFreqInclusive;
float PixelSize = (float)CTF.PixelSize;
float PixelDelta = (float)CTF.PixelSizeDelta;
float PixelAngle = (float)CTF.PixelSizeAngle * Helper.ToRad;
#endregion
#region Allocate GPU memory
Image CTFSpectra = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.X, NSpectra), true, true);
Image CTFCoordsCart = new Image(new int3(DimsRegion), true, true);
Image ParticleRefs = refft.AsProjections(ParticleAngles, DimsRegion, MainWindow.Options.ProjectionOversample);
/*Image ParticleRefsIFT = ParticleRefs.AsIFFT();
ParticleRefsIFT.WriteMRC("d_particlerefs.mrc");
ParticleRefsIFT.Dispose();*/
#endregion
// Extract movie regions, create individual spectra in Cartesian coordinates.
#region Create spectra
Image ParticleMasksFT = maskft.AsProjections(ParticleAngles, DimsRegion, MainWindow.Options.ProjectionOversample);
Image ParticleMasks = ParticleMasksFT.AsIFFT();
ParticleMasksFT.Dispose();
Parallel.ForEach(ParticleMasks.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(25f, slice[i])) - 2) / 23f;
});
int3[] PositionsExtractionPerFrame = new int3[PositionsExtraction.Length * NFrames];
for (int z = 0; z < NFrames; z++)
{
for (int p = 0; p < NPositions; p++)
{
float3 Coords = new float3(PositionsGrid[p].X, PositionsGrid[p].Y, z / (float)(NFrames - 1));
float2 Offset = GetShiftFromPyramid(Coords);
PositionsExtractionPerFrame[z * NPositions + p] = new int3((int)Math.Round(PositionsExtraction[p].X - Offset.X),
(int)Math.Round(PositionsExtraction[p].Y - Offset.Y),
0);
}
}
float3[] PositionsGridPerFrame = new float3[NSpectra];
for (int z = 0; z < NFrameGroups; z++)
{
for (int p = 0; p < NPositions; p++)
{
float3 Coords = new float3(PositionsGrid[p].X, PositionsGrid[p].Y, (z * FrameGroupSize + FrameGroupSize / 2) / (float)(NFrames - 1));
PositionsGridPerFrame[z * NPositions + p] = Coords;
}
}
GPU.CreateParticleSpectra(originalStack.GetDevice(Intent.Read),
DimsImage,
NFrames,
PositionsExtractionPerFrame,
NPositions,
ParticleMasks.GetDevice(Intent.Read),
DimsRegion,
CTFGridZ > 1,
FrameGroupSize,
PixelSize + PixelDelta / 2f,
PixelSize - PixelDelta / 2f,
PixelAngle,
CTFSpectra.GetDevice(Intent.Write));
originalStack.FreeDevice(); // Won't need it in this method anymore.
ParticleMasks.Dispose();
/*Image CTFSpectraIFT = CTFSpectra.AsIFFT();
CTFSpectraIFT.RemapFromFT();
CTFSpectraIFT.WriteMRC("d_ctfspectra.mrc");
CTFSpectraIFT.Dispose();*/
#endregion
// Populate address arrays for later.
#region Init addresses
{
float2[] CoordsData = new float2[CTFCoordsCart.ElementsSliceComplex];
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) => CoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r / DimsRegion.X, a));
CTFCoordsCart.UpdateHostWithComplex(new[] { CoordsData });
CTFCoordsCart.RemapToFT();
}
#endregion
// Band-pass filter reference projections
{
Image BandMask = new Image(new int3(DimsRegion.X, DimsRegion.Y, 1), true);
float[] BandMaskData = BandMask.GetHost(Intent.Write)[0];
float[] CTFCoordsData = CTFCoordsCart.GetHost(Intent.Read)[0];
for (int i = 0; i < BandMaskData.Length; i++)
BandMaskData[i] = (CTFCoordsData[i * 2] >= MinFreqInclusive / (float)DimsRegion.X && CTFCoordsData[i * 2] < MaxFreqExclusive / (float)DimsRegion.X) ? 1 : 0;
ParticleRefs.MultiplySlices(BandMask);
BandMask.Dispose();
}
Image Sigma2Noise = new Image(new int3(DimsRegion), true);
{
int GroupNumber = int.Parse(stardata.GetRowValue(RowIndices[0], "rlnGroupNumber"));
Star SigmaTable = new Star(MainWindow.Options.ModelStarPath, "data_model_group_" + GroupNumber);
float[] SigmaValues = SigmaTable.GetColumn("rlnSigma2Noise").Select(v => float.Parse(v)).ToArray();
float[] Sigma2NoiseData = Sigma2Noise.GetHost(Intent.Write)[0];
Helper.ForEachElementFT(new int2(DimsRegion.X, DimsRegion.Y), (x, y, xx, yy, r, angle) =>
{
int ir = (int)r;
float val = 0;
if (ir < SigmaValues.Length)
{
if (SigmaValues[ir] != 0f)
val = 1f / SigmaValues[ir];
}
Sigma2NoiseData[y * (DimsRegion.X / 2 + 1) + x] = val;
});
float MaxSigma = MathHelper.Max(Sigma2NoiseData);
for (int i = 0; i < Sigma2NoiseData.Length; i++)
Sigma2NoiseData[i] /= MaxSigma;
Sigma2Noise.RemapToFT();
}
Sigma2Noise.WriteMRC("d_sigma2noise.mrc");
// Do BFGS optimization of defocus, astigmatism and phase shift,
// using 2D simulation for comparison
#region BFGS
{
// Wiggle weights show how the defocus on the spectra grid is altered
// by changes in individual anchor points of the spline grid.
// They are used later to compute the dScore/dDefocus values for each spectrum
// only once, and derive the values for each anchor point from them.
float[][] WiggleWeights = GridCTF.GetWiggleWeights(PositionsGridPerFrame);
float[][] WiggleWeightsPhase = GridCTFPhase.GetWiggleWeights(PositionsGridPerFrame);
// Helper method for getting CTFStructs for the entire spectra grid.
Func<double[], CTF, float[], float[], CTFStruct[]> EvalGetCTF = (input, ctf, defocusValues, phaseValues) =>
{
decimal AlteredDelta = (decimal)input[input.Length - 2];
decimal AlteredAngle = (decimal)(input[input.Length - 1] * 20 / (Math.PI / 180));
CTF Local = ctf.GetCopy();
Local.DefocusDelta = AlteredDelta;
Local.DefocusAngle = AlteredAngle;
Local.PixelSizeDelta = 0;
CTFStruct LocalStruct = Local.ToStruct();
CTFStruct[] LocalParams = new CTFStruct[defocusValues.Length];
for (int i = 0; i < LocalParams.Length; i++)
{
LocalParams[i] = LocalStruct;
LocalParams[i].Defocus = defocusValues[i] * -1e-6f;
LocalParams[i].PhaseShift = phaseValues[i] * (float)Math.PI;
}
return LocalParams;
};
// Simulate with adjusted CTF, compare to originals
#region Eval and Gradient methods
Func<double[], double> Eval = input =>
{
CubicGrid Altered = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] DefocusValues = Altered.GetInterpolatedNative(PositionsGridPerFrame);
CubicGrid AlteredPhase = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] PhaseValues = AlteredPhase.GetInterpolatedNative(PositionsGridPerFrame);
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
float[] Result = new float[LocalParams.Length];
GPU.ParticleCTFCompareToSim(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
ParticleRefs.GetDevice(Intent.Read),
Sigma2Noise.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceComplex,
LocalParams,
Result,
(uint)NFrameGroups,
(uint)NPositions);
float Score = 0;
for (int i = 0; i < Result.Length; i++)
Score += Result[i];
Score /= NSpectra;
if (float.IsNaN(Score) || float.IsInfinity(Score))
throw new Exception("Bad score.");
return Score * 1.0;
};
Func<double[], double[]> Gradient = input =>
{
const float Step = 0.001f;
double[] Result = new double[input.Length];
// In 0D grid case, just get gradient for all 4 parameters.
// In 1+D grid case, do simple gradient for astigmatism and phase...
int StartComponent = input.Length - 2;
//int StartComponent = 0;
/*for (int i = StartComponent; i < input.Length; i++)
{
double[] UpperInput = new double[input.Length];
input.CopyTo(UpperInput, 0);
UpperInput[i] += Step;
double UpperValue = Eval(UpperInput);
double[] LowerInput = new double[input.Length];
input.CopyTo(LowerInput, 0);
LowerInput[i] -= Step;
double LowerValue = Eval(LowerInput);
Result[i] = (UpperValue - LowerValue) / (2f * Step);
}*/
float[] ResultPlus = new float[NSpectra];
float[] ResultMinus = new float[NSpectra];
// ..., take shortcut for defoci...
{
CubicGrid AlteredPhase = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] PhaseValues = AlteredPhase.GetInterpolatedNative(PositionsGridPerFrame);
{
CubicGrid AlteredPlus = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v + Step).ToArray());
float[] DefocusValues = AlteredPlus.GetInterpolatedNative(PositionsGridPerFrame);
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.ParticleCTFCompareToSim(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
ParticleRefs.GetDevice(Intent.Read),
Sigma2Noise.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceComplex,
LocalParams,
ResultPlus,
(uint)NFrameGroups,
(uint)NPositions);
}
{
CubicGrid AlteredMinus = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v - Step).ToArray());
float[] DefocusValues = AlteredMinus.GetInterpolatedNative(PositionsGridPerFrame);
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.ParticleCTFCompareToSim(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
ParticleRefs.GetDevice(Intent.Read),
Sigma2Noise.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceComplex,
LocalParams,
ResultMinus,
(uint)NFrameGroups,
(uint)NPositions);
}
float[] LocalGradients = new float[ResultPlus.Length];
for (int i = 0; i < LocalGradients.Length; i++)
LocalGradients[i] = ResultPlus[i] - ResultMinus[i];
// Now compute gradients per grid anchor point using the precomputed individual gradients and wiggle factors.
Parallel.For(0, GridCTF.Dimensions.Elements(), i => Result[i] = MathHelper.ReduceWeighted(LocalGradients, WiggleWeights[i]) / LocalGradients.Length / (2f * Step) * 1f);
}
// ..., and take shortcut for phases.
if (MainWindow.Options.CTFDoPhase)
{
CubicGrid AlteredPlus = new CubicGrid(GridCTF.Dimensions, input.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
float[] DefocusValues = AlteredPlus.GetInterpolatedNative(PositionsGridPerFrame);
{
CubicGrid AlteredPhasePlus = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v + Step).ToArray());
float[] PhaseValues = AlteredPhasePlus.GetInterpolatedNative(PositionsGridPerFrame);
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.ParticleCTFCompareToSim(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
ParticleRefs.GetDevice(Intent.Read),
Sigma2Noise.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceComplex,
LocalParams,
ResultPlus,
(uint)NFrameGroups,
(uint)NPositions);
}
{
CubicGrid AlteredPhaseMinus = new CubicGrid(GridCTFPhase.Dimensions, input.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v - Step).ToArray());
float[] PhaseValues = AlteredPhaseMinus.GetInterpolatedNative(PositionsGridPerFrame);
CTFStruct[] LocalParams = EvalGetCTF(input, CTF, DefocusValues, PhaseValues);
GPU.ParticleCTFCompareToSim(CTFSpectra.GetDevice(Intent.Read),
CTFCoordsCart.GetDevice(Intent.Read),
ParticleRefs.GetDevice(Intent.Read),
Sigma2Noise.GetDevice(Intent.Read),
(uint)CTFSpectra.ElementsSliceComplex,
LocalParams,
ResultMinus,
(uint)NFrameGroups,
(uint)NPositions);
}
float[] LocalGradients = new float[ResultPlus.Length];
for (int i = 0; i < LocalGradients.Length; i++)
LocalGradients[i] = ResultPlus[i] - ResultMinus[i];
// Now compute gradients per grid anchor point using the precomputed individual gradients and wiggle factors.
Parallel.For(0, GridCTFPhase.Dimensions.Elements(), i => Result[i + GridCTF.Dimensions.Elements()] = MathHelper.ReduceWeighted(LocalGradients, WiggleWeightsPhase[i]) / LocalGradients.Length / (2f * Step) * 1f);
}
foreach (var i in Result)
if (double.IsNaN(i) || double.IsInfinity(i))
throw new Exception("Bad score.");
return Result;
};
#endregion
#region Maximize normalized cross-correlation
double[] StartParams = new double[GridCTF.Dimensions.Elements() + GridCTFPhase.Dimensions.Elements() + 2];
for (int i = 0; i < GridCTF.Dimensions.Elements(); i++)
StartParams[i] = GridCTF.FlatValues[i];
for (int i = 0; i < GridCTFPhase.Dimensions.Elements(); i++)
StartParams[i + GridCTF.Dimensions.Elements()] = GridCTFPhase.FlatValues[i];
StartParams[StartParams.Length - 2] = (double)CTF.DefocusDelta;
StartParams[StartParams.Length - 1] = (double)CTF.DefocusAngle / 20 * (Math.PI / 180);
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Gradient);
/*{
Past = 1,
Delta = 1e-6,
MaxLineSearch = 15,
Corrections = 20
};*/
double[] BestStart = new double[StartParams.Length];
for (int i = 0; i < BestStart.Length; i++)
BestStart[i] = StartParams[i];
double BestValue = Eval(StartParams);
for (int o = 0; o < 1; o++)
{
/*for (int step = 0; step < 150; step++)
{
float Adjustment = (step - 75) / 75f * 0.075f;
double[] Adjusted = new double[StartParams.Length];
for (int j = 0; j < Adjusted.Length; j++)
if (j < GridCTF.Dimensions.Elements())
Adjusted[j] = StartParams[j] + Adjustment;
else
Adjusted[j] = StartParams[j];
double NewValue = Eval(Adjusted);
if (NewValue > BestValue)
{
BestValue = NewValue;
for (int j = 0; j < GridCTF.Dimensions.Elements(); j++)
BestStart[j] = StartParams[j] + Adjustment;
}
}
for (int i = 0; i < GridCTF.Dimensions.Elements(); i++)
StartParams[i] = BestStart[i];*/
Optimizer.Maximize(StartParams);
}
#endregion
#region Retrieve parameters
decimal NewDefocus = (decimal)MathHelper.Mean(StartParams.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v));
Debug.WriteLine(CTF.Defocus - NewDefocus);
CTF.Defocus = (decimal)MathHelper.Mean(StartParams.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v));
CTF.DefocusDelta = (decimal)StartParams[StartParams.Length - 2];
CTF.DefocusAngle = (decimal)(StartParams[StartParams.Length - 1] * 20 / (Math.PI / 180));
CTF.PhaseShift = (decimal)MathHelper.Mean(StartParams.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v));
GridCTF = new CubicGrid(GridCTF.Dimensions, StartParams.Take((int)GridCTF.Dimensions.Elements()).Select(v => (float)v).ToArray());
GridCTFPhase = new CubicGrid(GridCTFPhase.Dimensions, StartParams.Skip((int)GridCTF.Dimensions.Elements()).Take((int)GridCTFPhase.Dimensions.Elements()).Select(v => (float)v).ToArray());
#endregion
Sigma2Noise.Dispose();
}
#endregion
ParticleRefs.Dispose();
//ParticleAmps.Dispose();
CTFSpectra.Dispose();
CTFCoordsCart.Dispose();
Simulated1D = GetSimulated1D();
CTFQuality = GetCTFQuality();
SaveMeta();
}