public void ExportSubtomos(Star tableIn, Image tiltStack, int size, int3 volumeDimensions)
{
VolumeDimensions = volumeDimensions;
#region Get rows from table
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName + "."))
RowIndices.Add(i);
//if (RowIndices.Count == 0)
// return;
int NParticles = RowIndices.Count;
#endregion
#region Make sure all columns and directories are there
if (!tableIn.HasColumn("rlnImageName"))
tableIn.AddColumn("rlnImageName");
if (!tableIn.HasColumn("rlnCtfImage"))
tableIn.AddColumn("rlnCtfImage");
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
#endregion
#region Get subtomo positions from table
float3[] Origins = new float3[NParticles];
float3[] ParticleAngles = new float3[NParticles];
{
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnPosZ = tableIn.GetColumn("rlnCoordinateZ");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
string[] ColumnOriginZ = tableIn.GetColumn("rlnOriginZ");
string[] ColumnAngleRot = tableIn.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = tableIn.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = tableIn.GetColumn("rlnAnglePsi");
for (int i = 0; i < NParticles; i++)
{
float3 Pos = new float3(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosZ[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Shift = new float3(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginZ[RowIndices[i]], CultureInfo.InvariantCulture));
//Origins[i] *= new float3(3838f / 959f, 3710f / 927f, 4f);
Origins[i] = Pos - Shift;
float3 Angle = new float3(0, 0, 0);
if (ColumnAngleRot != null && ColumnAngleTilt != null && ColumnAnglePsi != null)
Angle = new float3(float.Parse(ColumnAngleRot[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAngleTilt[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAnglePsi[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleAngles[i] = Angle;
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", Origins[i].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", Origins[i].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateZ", Origins[i].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginZ", "0.0");
//tableIn.SetRowValue(RowIndices[i], "rlnAngleRot", "0.0");
//tableIn.SetRowValue(RowIndices[i], "rlnAngleTilt", "0.0");
//tableIn.SetRowValue(RowIndices[i], "rlnAnglePsi", "0.0");
}
}
#endregion
tiltStack.FreeDevice();
/*int SizeCropped = 50;//size / 2;
int3 Grid = (VolumeDimensions + SizeCropped - 1) / SizeCropped;
List<float3> GridCoords = new List<float3>();
for (int z = 0; z < Grid.Z; z++)
for (int y = 0; y < Grid.Y; y++)
for (int x = 0; x < Grid.X; x++)
GridCoords.Add(new float3(x * SizeCropped + SizeCropped / 2,
y * SizeCropped + SizeCropped / 2,
z * SizeCropped + SizeCropped / 2));
Origins = GridCoords.ToArray();
NParticles = Origins.Length;*/
if (NParticles == 0)
return;
Image CTFCoords = GetCTFCoords(size, size);
int PlanForw, PlanBack, PlanForwCTF;
Projector.GetPlans(new int3(size, size, size), 2, out PlanForw, out PlanBack, out PlanForwCTF);
//Parallel.For(0, NParticles, new ParallelOptions() { MaxDegreeOfParallelism = 1 }, p =>
for (int p = 0; p < NParticles; p++)
{
lock (tableIn)
{
tableIn.SetRowValue(RowIndices[p], "rlnImageName", "particles/" + RootName + "_" + p.ToString("D5") + ".mrc");
tableIn.SetRowValue(RowIndices[p], "rlnCtfImage", "particlectf/" + RootName + "_" + p.ToString("D5") + ".mrc");
//tableIn.Save("D:\\rubisco\\luisexported.star");
}
if (File.Exists(ParticlesDir + RootName + "_" + p.ToString("D5") + ".mrc"))
return;
Image Subtomo, SubtomoCTF;
GetSubtomo(tiltStack, Origins[p], ParticleAngles[p], CTFCoords, out Subtomo, out SubtomoCTF, PlanForw, PlanBack, PlanForwCTF);
//Image SubtomoCropped = Subtomo.AsPadded(new int3(SizeCropped, SizeCropped, SizeCropped));
Subtomo.WriteMRC(ParticlesDir + RootName + "_" + p.ToString("D5") + ".mrc");
SubtomoCTF.WriteMRC(ParticleCTFDir + RootName + "_" + p.ToString("D5") + ".mrc");
//SubtomoCropped.Dispose();
Subtomo?.Dispose();
SubtomoCTF?.Dispose();
}//);
GPU.DestroyFFTPlan(PlanForw);
GPU.DestroyFFTPlan(PlanBack);
GPU.DestroyFFTPlan(PlanForwCTF);
CTFCoords.Dispose();
}
public void RealspaceRefineGlobal(Star tableIn, Image tiltStack, int size, int3 volumeDimensions, Dictionary<int, Projector> references, float resolution, int healpixorder, string symmetry, Dictionary<int, Projector> outReconstructions)
{
VolumeDimensions = volumeDimensions;
#region Get rows from table
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return;
int NParticles = RowIndices.Count;
#endregion
#region Make sure all columns and directories are there
if (!tableIn.HasColumn("rlnImageName"))
tableIn.AddColumn("rlnImageName");
if (!tableIn.HasColumn("rlnCtfImage"))
tableIn.AddColumn("rlnCtfImage");
if (!tableIn.HasColumn("rlnParticleSelectZScore"))
tableIn.AddColumn("rlnParticleSelectZScore");
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
#endregion
#region Get subtomo positions from table
float3[] ParticleOrigins = new float3[NParticles];
float3[] ParticleAngles = new float3[NParticles];
int[] ParticleSubset = new int[NParticles];
{
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnPosZ = tableIn.GetColumn("rlnCoordinateZ");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
string[] ColumnOriginZ = tableIn.GetColumn("rlnOriginZ");
string[] ColumnAngleRot = tableIn.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = tableIn.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = tableIn.GetColumn("rlnAnglePsi");
string[] ColumnSubset = tableIn.GetColumn("rlnRandomSubset");
for (int i = 0; i < NParticles; i++)
{
float3 Pos = new float3(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosZ[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Shift = new float3(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginZ[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleOrigins[i] = Pos - Shift;
//ParticleOrigins[i] /= new float3(3838f / 959f, 3710f / 927f, 4f);
float3 Angle = new float3(float.Parse(ColumnAngleRot[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAngleTilt[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAnglePsi[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleAngles[i] = Angle;
if (ColumnSubset != null)
ParticleSubset[i] = int.Parse(ColumnSubset[RowIndices[i]]);
else
ParticleSubset[i] = (i % 2) + 1;
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", ParticleOrigins[i].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", ParticleOrigins[i].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateZ", ParticleOrigins[i].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginZ", "0.0");
}
}
#endregion
#region Deal with subsets
List<int> SubsetIDs = new List<int>();
foreach (var i in ParticleSubset)
if (!SubsetIDs.Contains(i))
SubsetIDs.Add(i);
SubsetIDs.Sort();
// For each subset, create a list of its particle IDs
Dictionary<int, List<int>> SubsetParticleIDs = SubsetIDs.ToDictionary(subsetID => subsetID, subsetID => new List<int>());
for (int i = 0; i < ParticleSubset.Length; i++)
SubsetParticleIDs[ParticleSubset[i]].Add(i);
foreach (var list in SubsetParticleIDs.Values)
list.Sort();
// Note where each subset starts and ends in a unified, sorted (by subset) particle ID list
Dictionary<int, Tuple<int, int>> SubsetRanges = new Dictionary<int, Tuple<int, int>>();
{
int Start = 0;
foreach (var pair in SubsetParticleIDs)
{
SubsetRanges.Add(pair.Key, new Tuple<int, int>(Start, Start + pair.Value.Count));
Start += pair.Value.Count;
}
}
List<int> SubsetContinuousIDs = new List<int>();
foreach (var pair in SubsetParticleIDs)
SubsetContinuousIDs.AddRange(pair.Value);
// Reorder particle information to match the order of SubsetContinuousIDs
ParticleOrigins = SubsetContinuousIDs.Select(i => ParticleOrigins[i]).ToArray();
ParticleAngles = SubsetContinuousIDs.Select(i => ParticleAngles[i]).ToArray();
ParticleSubset = SubsetContinuousIDs.Select(i => ParticleSubset[i]).ToArray();
#endregion
int CoarseSize = (int)Math.Round(size * ((float)CTF.PixelSize * 2 / resolution)) / 2 * 2;
int3 CoarseDims = new int3(CoarseSize, CoarseSize, 1);
// Create mask, CTF coords, reference projections, shifts
Image Mask;
Image CTFCoords = GetCTFCoords(CoarseSize, size);
Dictionary<int, Image> SubsetProjections = new Dictionary<int, Image>();
float3[] AnglesOri = Helper.GetHealpixAngles(healpixorder, symmetry);
List<float3> Shifts = new List<float3>();
for (int z = -1; z <= 1; z++)
for (int y = -1; y <= 1; y++)
for (int x = -1; x <= 1; x++)
if (x * x + y * y + z * z <= 1)
Shifts.Add(new float3((float)x / CoarseSize * size * 0.5f, (float)y / CoarseSize * size * 0.5f, (float)z / CoarseSize * size * 0.5f));
#region Preflight
{
#region Create mask with soft edge
{
Image MaskBig = new Image(new int3(size, size, 1));
float MaskRadius = MainWindow.Options.ExportParticleRadius / (float)CTF.PixelSize;
float SoftEdge = 16f;
float[] MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= MaskRadius)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = (float)(Math.Cos(Math.Min(1, (R - MaskRadius) / SoftEdge) * Math.PI) * 0.5 + 0.5);
}
}
MaskBig.WriteMRC("d_maskbig.mrc");
Mask = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
Mask.RemapToFT();
MaskBig.Dispose();
}
Mask.WriteMRC("d_masksmall.mrc");
#endregion
#region Create projections for each angular sample, adjusted for each tilt
foreach (var subset in SubsetRanges)
{
float3[] AnglesAlt = new float3[AnglesOri.Length * NTilts];
for (int t = 0; t < NTilts; t++)
{
for (int a = 0; a < AnglesOri.Length; a++)
{
float GridStep = 1f / (NTilts - 1);
float3 GridCoords = new float3(0.5f, 0.5f, t * GridStep);
Matrix3 ParticleMatrix = Matrix3.Euler(AnglesOri[a].X * Helper.ToRad,
AnglesOri[a].Y * Helper.ToRad,
AnglesOri[a].Z * Helper.ToRad);
Matrix3 TiltMatrix = Matrix3.Euler(0, -AnglesCorrect[t] * Helper.ToRad, 0);
Matrix3 CorrectionMatrix = Matrix3.RotateZ(GridAngleZ.GetInterpolated(GridCoords) * Helper.ToRad) *
Matrix3.RotateY(GridAngleY.GetInterpolated(GridCoords) * Helper.ToRad) *
Matrix3.RotateX(GridAngleX.GetInterpolated(GridCoords) * Helper.ToRad);
Matrix3 Rotation = CorrectionMatrix * TiltMatrix * ParticleMatrix;
AnglesAlt[a * NTilts + t] = Matrix3.EulerFromMatrix(Rotation);
}
}
Image ProjFT = references[subset.Key].Project(new int2(CoarseSize, CoarseSize), AnglesAlt, CoarseSize / 2);
Image CheckProj = ProjFT.AsIFFT();
CheckProj.RemapFromFT();
CheckProj.WriteMRC("d_proj.mrc");
CheckProj.Dispose();
ProjFT.FreeDevice();
SubsetProjections[subset.Key] = ProjFT;
}
#endregion
}
#endregion
float3[] OptimizedShifts = new float3[NParticles];
float3[] OptimizedAngles = new float3[NParticles];
#region Correlate each particle with all projections
foreach (var subset in SubsetRanges)
{
Image Projections = SubsetProjections[subset.Key];
for (int p = subset.Value.Item1; p < subset.Value.Item2; p++)
//Parallel.For(subset.Value.Item1, subset.Value.Item2, new ParallelOptions { MaxDegreeOfParallelism = 4 }, p =>
{
Image ParticleImages;
Image ParticleCTFs = new Image(new int3(CoarseSize, CoarseSize, NTilts), true);
Image ParticleWeights;
// Positions the particles were extracted at/shifted to, to calculate effectively needed shifts later
float3[] ExtractedAt = new float3[NTilts];
float3 ParticleCoords = ParticleOrigins[p];
float3[] Positions = GetPositionInImages(ParticleCoords);
#region Create CTFs
{
float GridStep = 1f / (NTilts - 1);
CTFStruct[] Params = new CTFStruct[NTilts];
for (int t = 0; t < NTilts; t++)
{
decimal Defocus = (decimal)Positions[t].Z;
decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
CTF CurrCTF = CTF.GetCopy();
CurrCTF.Defocus = Defocus;
CurrCTF.DefocusDelta = DefocusDelta;
CurrCTF.DefocusAngle = DefocusAngle;
CurrCTF.Scale = (decimal)Math.Cos(Angles[t] * Helper.ToRad);
CurrCTF.Bfactor = (decimal)-Dose[t] * 8;
Params[t] = CurrCTF.ToStruct();
}
GPU.CreateCTF(ParticleCTFs.GetDevice(Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CoarseDims.ElementsFFT(),
Params,
false,
(uint)NTilts);
}
#endregion
#region Weights are sums of the 2D CTFs
{
Image CTFAbs = new Image(ParticleCTFs.GetDevice(Intent.Read), ParticleCTFs.Dims, true);
CTFAbs.Abs();
ParticleWeights = CTFAbs.AsSum2D();
CTFAbs.Dispose();
{
float[] Weights = ParticleWeights.GetHost(Intent.ReadWrite)[0];
float Sum = Weights.Sum();
for (int i = 0; i < Weights.Length; i++)
Weights[i] /= Sum;
}
}
#endregion
#region Extract images
{
Image Extracted = new Image(new int3(size, size, NTilts));
float[][] ExtractedData = Extracted.GetHost(Intent.Write);
Parallel.For(0, NTilts, t =>
{
ExtractedAt[t] = new float3((int)Positions[t].X, (int)Positions[t].Y, 0);
Positions[t] -= size / 2;
int2 IntPosition = new int2((int)Positions[t].X, (int)Positions[t].Y);
float[] OriginalData;
lock (tiltStack)
OriginalData = tiltStack.GetHost(Intent.Read)[t];
float[] ImageData = ExtractedData[t];
for (int y = 0; y < size; y++)
{
int PosY = (y + IntPosition.Y + tiltStack.Dims.Y) % tiltStack.Dims.Y;
for (int x = 0; x < size; x++)
{
int PosX = (x + IntPosition.X + tiltStack.Dims.X) % tiltStack.Dims.X;
ImageData[y * size + x] = -OriginalData[PosY * tiltStack.Dims.X + PosX];
}
}
});
ParticleImages = Extracted.AsScaled(new int2(CoarseSize, CoarseSize));
ParticleImages.RemapToFT();
//Scaled.WriteMRC("d_particleimages.mrc");
Extracted.Dispose();
}
#endregion
Image ProjectionsConv = new Image(IntPtr.Zero, Projections.Dims, true, true);
GPU.MultiplyComplexSlicesByScalar(Projections.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ProjectionsConv.GetDevice(Intent.Write),
ParticleCTFs.ElementsComplex,
(uint)AnglesOri.Length);
Image ProjectionsReal = ProjectionsConv.AsIFFT();
GPU.NormalizeMasked(ProjectionsReal.GetDevice(Intent.Read),
ProjectionsReal.GetDevice(Intent.Write),
Mask.GetDevice(Intent.Read),
(uint)ProjectionsReal.ElementsSliceReal,
(uint)ProjectionsReal.Dims.Z);
//ProjectionsReal.WriteMRC("d_projconv.mrc");
ProjectionsConv.Dispose();
#region For each particle offset, correlate each tilt image with all reference projection
float[][] AllResults = new float[Shifts.Count][];
//for (int s = 0; s < Shifts.Count; s++)
Parallel.For(0, Shifts.Count, new ParallelOptions { MaxDegreeOfParallelism = 2 }, s =>
{
AllResults[s] = new float[AnglesOri.Length];
float3 ParticleCoordsAlt = ParticleOrigins[p] - Shifts[s];
float3[] PositionsAlt = GetPositionInImages(ParticleCoordsAlt);
float3[] PositionDiff = new float3[NTilts];
for (int t = 0; t < NTilts; t++)
PositionDiff[t] = (ExtractedAt[t] - PositionsAlt[t]) / size * CoarseSize;
float[] ShiftResults = new float[AnglesOri.Length];
GPU.TomoRealspaceCorrelate(ProjectionsReal.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
(uint)AnglesOri.Length,
(uint)NTilts,
ParticleImages.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
Mask.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
Helper.ToInterleaved(PositionDiff),
ShiftResults);
AllResults[s] = ShiftResults;
});
#endregion
ProjectionsReal.Dispose();
ParticleImages.Dispose();
ParticleCTFs.Dispose();
ParticleWeights.Dispose();
#region Find best offset/angle combination and store it in table
float3 BestShift = new float3(0, 0, 0);
float3 BestAngle = new float3(0, 0, 0);
float BestScore = -1e30f;
for (int s = 0; s < Shifts.Count; s++)
for (int a = 0; a < AnglesOri.Length; a++)
if (AllResults[s][a] > BestScore)
{
BestScore = AllResults[s][a];
BestAngle = AnglesOri[a];
BestShift = Shifts[s];
}
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnOriginX", BestShift.X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnOriginY", BestShift.Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnOriginZ", BestShift.Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnAngleRot", BestAngle.X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnAngleTilt", BestAngle.Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnAnglePsi", BestAngle.Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[SubsetContinuousIDs[p]], "rlnParticleSelectZScore", BestScore.ToString(CultureInfo.InvariantCulture));
OptimizedShifts[p] = BestShift;
OptimizedAngles[p] = BestAngle;
#endregion
}
// Dispose all projections for this subset, they won't be needed later
SubsetProjections[subset.Key].Dispose();
}
#endregion
CTFCoords.Dispose();
#region Back-project with hopefully better parameters
{
CTFCoords = GetCTFCoords(size, size);
foreach (var subsetRange in SubsetRanges)
{
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3 ParticleCoords = ParticleOrigins[p] - OptimizedShifts[p];
Image FullParticleImages = GetSubtomoImages(tiltStack, size, ParticleCoords, true);
Image FullParticleCTFs = GetSubtomoCTFs(ParticleCoords, CTFCoords);
//Image FullParticleCTFWeights = GetSubtomoCTFs(ParticleCoords, CTFCoords, true, true);
FullParticleImages.Multiply(FullParticleCTFs);
//FullParticleImages.Multiply(FullParticleCTFWeights);
FullParticleCTFs.Multiply(FullParticleCTFs);
float3[] FullParticleAngles = GetParticleAngleInImages(ParticleCoords, OptimizedAngles[p]);
outReconstructions[subsetRange.Key].BackProject(FullParticleImages, FullParticleCTFs, FullParticleAngles);
FullParticleImages.Dispose();
FullParticleCTFs.Dispose();
//FullParticleCTFWeights.Dispose();
}
}
CTFCoords.Dispose();
}
#endregion
}
public Image SimulateTiltSeries(Star tableIn, int3 stackDimensions, int size, int3 volumeDimensions, Dictionary<int, Projector> references, float resolution)
{
VolumeDimensions = volumeDimensions;
Image SimulatedStack = new Image(stackDimensions);
#region Get rows from table
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return SimulatedStack;
int NParticles = RowIndices.Count;
#endregion
#region Make sure all columns and directories are there
if (!tableIn.HasColumn("rlnImageName"))
tableIn.AddColumn("rlnImageName");
if (!tableIn.HasColumn("rlnCtfImage"))
tableIn.AddColumn("rlnCtfImage");
if (!tableIn.HasColumn("rlnParticleSelectZScore"))
tableIn.AddColumn("rlnParticleSelectZScore");
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
#endregion
#region Get subtomo positions from table
float3[] ParticleOrigins = new float3[NParticles];
float3[] ParticleAngles = new float3[NParticles];
int[] ParticleSubset = new int[NParticles];
{
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnPosZ = tableIn.GetColumn("rlnCoordinateZ");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
string[] ColumnOriginZ = tableIn.GetColumn("rlnOriginZ");
string[] ColumnAngleRot = tableIn.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = tableIn.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = tableIn.GetColumn("rlnAnglePsi");
string[] ColumnSubset = tableIn.GetColumn("rlnRandomSubset");
for (int i = 0; i < NParticles; i++)
{
float3 Pos = new float3(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosZ[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Shift = new float3(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginZ[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleOrigins[i] = Pos - Shift;
//ParticleOrigins[i] /= new float3(3838f / 959f, 3710f / 927f, 4f);
float3 Angle = new float3(float.Parse(ColumnAngleRot[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAngleTilt[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAnglePsi[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleAngles[i] = Angle;
ParticleSubset[i] = int.Parse(ColumnSubset[RowIndices[i]]);
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", ParticleOrigins[i].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", ParticleOrigins[i].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateZ", ParticleOrigins[i].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginZ", "0.0");
}
}
#endregion
#region Deal with subsets
List<int> SubsetIDs = new List<int>();
foreach (var i in ParticleSubset)
if (!SubsetIDs.Contains(i))
SubsetIDs.Add(i);
SubsetIDs.Sort();
// For each subset, create a list of its particle IDs
Dictionary<int, List<int>> SubsetParticleIDs = SubsetIDs.ToDictionary(subsetID => subsetID, subsetID => new List<int>());
for (int i = 0; i < ParticleSubset.Length; i++)
SubsetParticleIDs[ParticleSubset[i]].Add(i);
foreach (var list in SubsetParticleIDs.Values)
list.Sort();
// Note where each subset starts and ends in a unified, sorted (by subset) particle ID list
Dictionary<int, Tuple<int, int>> SubsetRanges = new Dictionary<int, Tuple<int, int>>();
{
int Start = 0;
foreach (var pair in SubsetParticleIDs)
{
SubsetRanges.Add(pair.Key, new Tuple<int, int>(Start, Start + pair.Value.Count));
Start += pair.Value.Count;
}
}
List<int> SubsetContinuousIDs = new List<int>();
foreach (var pair in SubsetParticleIDs)
SubsetContinuousIDs.AddRange(pair.Value);
// Reorder particle information to match the order of SubsetContinuousIDs
ParticleOrigins = SubsetContinuousIDs.Select(i => ParticleOrigins[i]).ToArray();
ParticleAngles = SubsetContinuousIDs.Select(i => ParticleAngles[i]).ToArray();
ParticleSubset = SubsetContinuousIDs.Select(i => ParticleSubset[i]).ToArray();
#endregion
int CoarseSize = (int)Math.Round(size * ((float)CTF.PixelSize * 2 / resolution)) / 2 * 2;
int3 CoarseDims = new int3(CoarseSize, CoarseSize, 1);
// Positions the particles were extracted at/shifted to, to calculate effectively needed shifts later
float3[] ExtractedAt = new float3[NParticles * NTilts];
// Extract images, mask and resize them, create CTFs
#region Preflight
{
Image CTFCoords = GetCTFCoords(CoarseSize, size);
#region Create mask with soft edge
Image Mask;
{
Image MaskBig = new Image(new int3(size, size, 1));
float MaskRadius = MainWindow.Options.ExportParticleRadius / (float)CTF.PixelSize;
float SoftEdge = 16f;
float[] MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= MaskRadius)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = (float)(Math.Cos(Math.Min(1, (R - MaskRadius) / SoftEdge) * Math.PI) * 0.5 + 0.5);
}
}
MaskBig.WriteMRC("d_maskbig.mrc");
Mask = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
MaskBig.Dispose();
}
Mask.WriteMRC("d_masksmall.mrc");
#endregion
#region Create Fourier space mask
Image FourierMask = new Image(CoarseDims, true);
{
float[] FourierMaskData = FourierMask.GetHost(Intent.Write)[0];
int MaxR2 = CoarseSize * CoarseSize / 4;
for (int y = 0; y < CoarseSize; y++)
{
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
yy *= yy;
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x * x;
int R2 = yy + xx;
FourierMaskData[y * (CoarseSize / 2 + 1) + x] = R2 < MaxR2 ? 1 : 0;
}
}
}
#endregion
#region For each particle, create CTFs and projections, and insert them into the simulated tilt series
for (int p = 0; p < NParticles; p++)
{
Image ParticleImages;
Image ParticleCTFs = new Image(new int3(CoarseSize, CoarseSize, NTilts), true);
float3 ParticleCoords = ParticleOrigins[p];
float3[] Positions = GetPositionInImages(ParticleCoords);
// Create CTFs
{
float GridStep = 1f / (NTilts - 1);
CTFStruct[] Params = new CTFStruct[NTilts];
for (int t = 0; t < NTilts; t++)
{
decimal Defocus = (decimal)Positions[t].Z;
decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
CTF CurrCTF = CTF.GetCopy();
CurrCTF.Defocus = Defocus;
CurrCTF.DefocusDelta = DefocusDelta;
CurrCTF.DefocusAngle = DefocusAngle;
CurrCTF.Scale = (decimal)Math.Cos(Angles[t] * Helper.ToRad);
CurrCTF.Bfactor = (decimal)-Dose[t] * 8;
Params[t] = CurrCTF.ToStruct();
}
GPU.CreateCTF(ParticleCTFs.GetDevice(Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CoarseDims.ElementsFFT(),
Params,
false,
(uint)NTilts);
ParticleCTFs.MultiplySlices(FourierMask);
}
// Make projections
{
float3[] ImageShifts = new float3[NTilts];
float3[] ImageAngles = new float3[NTilts];
float3[] CurrPositions = GetPositionInImages(ParticleOrigins[p]);
float3[] CurrAngles = GetParticleAngleInImages(ParticleOrigins[p], ParticleAngles[p]);
for (int t = 0; t < NTilts; t++)
{
ImageShifts[t] = new float3(CurrPositions[t].X - (int)CurrPositions[t].X, // +diff because we are shifting the projections into experimental data frame
CurrPositions[t].Y - (int)CurrPositions[t].Y,
CurrPositions[t].Z - (int)CurrPositions[t].Z);
ImageAngles[t] = CurrAngles[t];
}
Image ProjectionsFT = new Image(IntPtr.Zero, new int3(CoarseSize, CoarseSize, NTilts), true, true);
Projector Reference = references[ParticleSubset[p]];
GPU.ProjectForward(Reference.Data.GetDevice(Intent.Read),
ProjectionsFT.GetDevice(Intent.Write),
Reference.Data.Dims,
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ImageAngles),
Reference.Oversampling,
(uint)NTilts);
ProjectionsFT.Multiply(ParticleCTFs);
Image Projections = ProjectionsFT.AsIFFT();
ProjectionsFT.Dispose();
Projections.RemapFromFT();
GPU.NormParticles(Projections.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Write),
new int3(CoarseSize, CoarseSize, 1),
(uint)(MainWindow.Options.ExportParticleRadius / CTF.PixelSize * (CTF.PixelSize * 2 / (decimal)resolution)),
true,
(uint)NTilts);
Projections.MultiplySlices(Mask);
ParticleImages = Projections.AsScaled(new int2(size, size));
Projections.Dispose();
ParticleImages.ShiftSlices(ImageShifts);
}
// Extract images
{
for (int t = 0; t < NTilts; t++)
{
Positions[t] -= size / 2;
int2 IntPosition = new int2((int)Positions[t].X, (int)Positions[t].Y);
float[] SimulatedData;
lock (SimulatedStack)
SimulatedData = SimulatedStack.GetHost(Intent.Write)[t];
float[] ImageData = ParticleImages.GetHost(Intent.Read)[t];
for (int y = 0; y < size; y++)
{
int PosY = (y + IntPosition.Y + SimulatedStack.Dims.Y) % SimulatedStack.Dims.Y;
for (int x = 0; x < size; x++)
{
int PosX = (x + IntPosition.X + SimulatedStack.Dims.X) % SimulatedStack.Dims.X;
SimulatedData[PosY * SimulatedStack.Dims.X + PosX] += ImageData[y * size + x];
}
}
}
}
ParticleImages.Dispose();
ParticleCTFs.Dispose();
}
#endregion
Mask.Dispose();
FourierMask.Dispose();
CTFCoords.Dispose();
}
#endregion
return SimulatedStack;
}
public void PerformGlobalParticleAlignment(Star tableIn,
Image tiltStack,
int size,
int3 volumeDimensions,
Dictionary<int, Projector> references,
float resolution,
int healpixOrder,
string symmetry,
float offsetRange,
float offsetStep,
Dictionary<int, Projector> outReconstructions,
Dictionary<int, Projector> outCTFReconstructions)
{
VolumeDimensions = volumeDimensions;
#region Get rows from table
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName + "."))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return;
int NParticles = RowIndices.Count;
#endregion
#region Make sure all columns and directories are there
if (!tableIn.HasColumn("rlnImageName"))
tableIn.AddColumn("rlnImageName");
if (!tableIn.HasColumn("rlnCtfImage"))
tableIn.AddColumn("rlnCtfImage");
if (!tableIn.HasColumn("rlnParticleSelectZScore"))
tableIn.AddColumn("rlnParticleSelectZScore");
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
#endregion
#region Get subtomo positions from table
float3[] ParticleOrigins = new float3[NParticles];
float3[] ParticleOrigins2 = new float3[NParticles];
float3[] ParticleAngles = new float3[NParticles];
float3[] ParticleAngles2 = new float3[NParticles];
int[] ParticleSubset = new int[NParticles];
{
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnPosZ = tableIn.GetColumn("rlnCoordinateZ");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
string[] ColumnOriginZ = tableIn.GetColumn("rlnOriginZ");
string[] ColumnAngleRot = tableIn.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = tableIn.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = tableIn.GetColumn("rlnAnglePsi");
string[] ColumnSubset = tableIn.GetColumn("rlnRandomSubset");
for (int i = 0; i < NParticles; i++)
{
float3 Pos = new float3(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosZ[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Pos2 = Pos;
float3 Shift = new float3(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginZ[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleOrigins[i] = Pos - Shift;
ParticleOrigins2[i] = Pos2 - Shift;
float3 Angle = new float3(float.Parse(ColumnAngleRot[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAngleTilt[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAnglePsi[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Angle2 = Angle;
ParticleAngles[i] = Angle;
ParticleAngles2[i] = Angle2;
ParticleSubset[i] = int.Parse(ColumnSubset[RowIndices[i]]);
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", ParticleOrigins[i].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", ParticleOrigins[i].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateZ", ParticleOrigins[i].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginZ", "0.0");
}
}
#endregion
#region Deal with subsets
List<int> SubsetIDs = new List<int>();
foreach (var i in ParticleSubset)
if (!SubsetIDs.Contains(i))
SubsetIDs.Add(i);
SubsetIDs.Sort();
// For each subset, create a list of its particle IDs
Dictionary<int, List<int>> SubsetParticleIDs = SubsetIDs.ToDictionary(subsetID => subsetID, subsetID => new List<int>());
for (int i = 0; i < ParticleSubset.Length; i++)
SubsetParticleIDs[ParticleSubset[i]].Add(i);
foreach (var list in SubsetParticleIDs.Values)
list.Sort();
// Note where each subset starts and ends in a unified, sorted (by subset) particle ID list
Dictionary<int, Tuple<int, int>> SubsetRanges = new Dictionary<int, Tuple<int, int>>();
{
int Start = 0;
foreach (var pair in SubsetParticleIDs)
{
SubsetRanges.Add(pair.Key, new Tuple<int, int>(Start, Start + pair.Value.Count));
Start += pair.Value.Count;
}
}
List<int> SubsetContinuousIDs = new List<int>();
foreach (var pair in SubsetParticleIDs)
SubsetContinuousIDs.AddRange(pair.Value);
// Reorder particle information to match the order of SubsetContinuousIDs
ParticleOrigins = SubsetContinuousIDs.Select(i => ParticleOrigins[i]).ToArray();
ParticleOrigins2 = SubsetContinuousIDs.Select(i => ParticleOrigins2[i]).ToArray();
ParticleAngles = SubsetContinuousIDs.Select(i => ParticleAngles[i]).ToArray();
ParticleAngles2 = SubsetContinuousIDs.Select(i => ParticleAngles2[i]).ToArray();
ParticleSubset = SubsetContinuousIDs.Select(i => ParticleSubset[i]).ToArray();
#endregion
int CoarseSize = (int)Math.Round(size * ((float)CTF.PixelSize * 2 / resolution)) / 2 * 2;
int3 CoarseDims = new int3(CoarseSize, CoarseSize, 1);
// Positions the particles were extracted at/shifted to, to calculate effectively needed shifts later
float2[] ExtractedAt = new float2[NParticles * NTilts];
// Extract images, mask and resize them, create CTFs
Image ParticleImages = new Image(new int3(CoarseSize, CoarseSize, NParticles * NTilts), true, true);
Image ParticleCTFs = new Image(new int3(CoarseSize, CoarseSize, NParticles * NTilts), true);
Image ParticleWeights = null;
Image ShiftFactors = null;
#region Preflight
float KeepBFac = GlobalBfactor;
GlobalBfactor = 0;
{
Image CTFCoords = GetCTFCoords(CoarseSize, size);
#region Precalculate vectors for shifts in Fourier space
{
float2[] ShiftFactorsData = new float2[(CoarseSize / 2 + 1) * CoarseSize];
for (int y = 0; y < CoarseSize; y++)
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x;
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
ShiftFactorsData[y * (CoarseSize / 2 + 1) + x] = new float2((float)-xx / size * 2f * (float)Math.PI,
(float)-yy / size * 2f * (float)Math.PI);
}
ShiftFactors = new Image(ShiftFactorsData, new int3(CoarseSize, CoarseSize, 1), true);
}
#endregion
#region Create mask with soft edge
Image Mask;
Image MaskSubt;
{
Image MaskBig = new Image(new int3(size, size, 1));
float MaskRadius = MainWindow.Options.ExportParticleRadius / (float)CTF.PixelSize;
float SoftEdge = 16f;
float[] MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= MaskRadius)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = (float)(Math.Cos(Math.Min(1, (R - MaskRadius) / SoftEdge) * Math.PI) * 0.5 + 0.5);
}
}
//MaskBig.WriteMRC("d_maskbig.mrc");
Mask = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
Mask.RemapToFT();
MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= 30)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = 0;
}
}
MaskSubt = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
MaskSubt.RemapToFT();
MaskBig.Dispose();
}
//Mask.WriteMRC("d_masksmall.mrc");
#endregion
#region Create Fourier space mask
Image FourierMask = new Image(CoarseDims, true);
{
float[] FourierMaskData = FourierMask.GetHost(Intent.Write)[0];
int MaxR2 = CoarseSize * CoarseSize / 4;
for (int y = 0; y < CoarseSize; y++)
{
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
yy *= yy;
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x * x;
int R2 = yy + xx;
FourierMaskData[y * (CoarseSize / 2 + 1) + x] = R2 < MaxR2 ? 1 : 0;
}
}
}
#endregion
#region For each particle, create CTFs and extract & preprocess images for entire tilt series
for (int p = 0; p < NParticles; p++)
{
float3 ParticleCoords = ParticleOrigins[p];
float3[] Positions = GetPositionInImages(ParticleCoords);
float3[] ProjAngles = GetParticleAngleInImages(ParticleCoords, ParticleAngles[p]);
Image Extracted = new Image(new int3(size, size, NTilts));
float[][] ExtractedData = Extracted.GetHost(Intent.Write);
float3[] Residuals = new float3[NTilts];
Image SubtrahendsCTF = new Image(new int3(CoarseSize, CoarseSize, NTilts), true);
// Create CTFs
{
CTFStruct[] CTFParams = new CTFStruct[NTilts];
float GridStep = 1f / (NTilts - 1);
CTFStruct[] Params = new CTFStruct[NTilts];
for (int t = 0; t < NTilts; t++)
{
decimal Defocus = (decimal)Positions[t].Z;
decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
CTF CurrCTF = CTF.GetCopy();
CurrCTF.Defocus = Defocus;
CurrCTF.DefocusDelta = DefocusDelta;
CurrCTF.DefocusAngle = DefocusAngle;
CurrCTF.Scale = (decimal)Math.Cos(Angles[t] * Helper.ToRad);
CurrCTF.Bfactor = (decimal)-Dose[t] * 8;
Params[t] = CurrCTF.ToStruct();
}
GPU.CreateCTF(ParticleCTFs.GetDeviceSlice(NTilts * p, Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CoarseDims.ElementsFFT(),
Params,
false,
(uint)NTilts);
}
// Extract images
{
for (int t = 0; t < NTilts; t++)
{
ExtractedAt[p * NTilts + t] = new float2(Positions[t].X, Positions[t].Y);
Positions[t] -= size / 2;
int2 IntPosition = new int2((int)Positions[t].X, (int)Positions[t].Y);
float2 Residual = new float2(-(Positions[t].X - IntPosition.X), -(Positions[t].Y - IntPosition.Y));
Residuals[t] = new float3(Residual / size * CoarseSize);
float[] OriginalData;
lock (tiltStack)
OriginalData = tiltStack.GetHost(Intent.Read)[t];
float[] ImageData = ExtractedData[t];
for (int y = 0; y < size; y++)
{
int PosY = (y + IntPosition.Y + tiltStack.Dims.Y) % tiltStack.Dims.Y;
for (int x = 0; x < size; x++)
{
int PosX = (x + IntPosition.X + tiltStack.Dims.X) % tiltStack.Dims.X;
ImageData[y * size + x] = OriginalData[PosY * tiltStack.Dims.X + PosX];
}
}
}
GPU.NormParticles(Extracted.GetDevice(Intent.Read),
Extracted.GetDevice(Intent.Write),
new int3(size, size, 1),
(uint)(MainWindow.Options.ExportParticleRadius / CTF.PixelSize),
true,
(uint)NTilts);
Image Scaled = Extracted.AsScaled(new int2(CoarseSize, CoarseSize));
//Scaled.WriteMRC("d_scaled.mrc");
Extracted.Dispose();
Scaled.ShiftSlices(Residuals);
Scaled.RemapToFT();
//GPU.NormalizeMasked(Scaled.GetDevice(Intent.Read),
// Scaled.GetDevice(Intent.Write),
// MaskSubt.GetDevice(Intent.Read),
// (uint)Scaled.ElementsSliceReal,
// (uint)NTilts);
//{
// //Image SubtrahendsFT = subtrahendReference.Project(new int2(CoarseSize, CoarseSize), ProjAngles, CoarseSize / 2);
// //SubtrahendsFT.Multiply(SubtrahendsCTF);
// //Image Subtrahends = SubtrahendsFT.AsIFFT();
// //SubtrahendsFT.Dispose();
// ////GPU.NormalizeMasked(Subtrahends.GetDevice(Intent.Read),
// //// Subtrahends.GetDevice(Intent.Write),
// //// MaskSubt.GetDevice(Intent.Read),
// //// (uint)Subtrahends.ElementsSliceReal,
// //// (uint)NTilts);
// //Scaled.Subtract(Subtrahends);
// //Subtrahends.Dispose();
// Image FocusMaskFT = maskReference.Project(new int2(CoarseSize, CoarseSize), ProjAngles, CoarseSize / 2);
// Image FocusMask = FocusMaskFT.AsIFFT();
// FocusMaskFT.Dispose();
// Scaled.Multiply(FocusMask);
// FocusMask.Dispose();
//}
Scaled.MultiplySlices(Mask);
GPU.FFT(Scaled.GetDevice(Intent.Read),
ParticleImages.GetDeviceSlice(p * NTilts, Intent.Write),
CoarseDims,
(uint)NTilts);
Scaled.Dispose();
SubtrahendsCTF.Dispose();
}
}
#endregion
ParticleCTFs.MultiplySlices(FourierMask);
Mask.Dispose();
FourierMask.Dispose();
MaskSubt.Dispose();
Image ParticleCTFsAbs = new Image(ParticleCTFs.GetDevice(Intent.Read), ParticleCTFs.Dims, true);
ParticleCTFsAbs.Abs();
ParticleWeights = ParticleCTFsAbs.AsSum2D();
ParticleCTFsAbs.Dispose();
{
float[] ParticleWeightsData = ParticleWeights.GetHost(Intent.ReadWrite)[0];
float Max = MathHelper.Max(ParticleWeightsData);
for (int i = 0; i < ParticleWeightsData.Length; i++)
ParticleWeightsData[i] /= Max;
}
CTFCoords.Dispose();
//Image CheckImages = ParticleImages.AsIFFT();
//CheckImages.WriteMRC("d_particleimages.mrc");
//CheckImages.Dispose();
//ParticleCTFs.WriteMRC("d_particlectfs.mrc");
}
GlobalBfactor = KeepBFac;
#endregion
#region Global alignment
Func<float3[], float2[]> GetImageShifts = input =>
{
// Using current positions, angles and grids, get parameters for image shifts
float2[] ImageShifts = new float2[NParticles * NTilts];
float3[] PerTiltPositions = new float3[NParticles * NTilts];
for (int p = 0; p < NParticles; p++)
for (int t = 0; t < NTilts; t++)
PerTiltPositions[p * NTilts + t] = input[p];
float3[] CurrPositions = GetPositionInImages(PerTiltPositions);
for (int i = 0; i < ImageShifts.Length; i++)
ImageShifts[i] = new float2(ExtractedAt[i].X - CurrPositions[i].X,
ExtractedAt[i].Y - CurrPositions[i].Y); // -diff because those are extraction positions, i. e. opposite direction of shifts
return ImageShifts;
};
Func<float3[], float3[]> GetImageAngles = input =>
{
int NAngles = input.Length;
float3 VolumeCenter = new float3(VolumeDimensions.X / 2, VolumeDimensions.Y / 2, VolumeDimensions.Z / 2);
float3[] PerTiltPositions = new float3[NAngles * NTilts];
float3[] PerTiltAngles = new float3[NAngles * NTilts];
for (int a = 0; a < NAngles; a++)
for (int t = 0; t < NTilts; t++)
{
PerTiltPositions[a * NTilts + t] = VolumeCenter;
PerTiltAngles[a * NTilts + t] = input[a];
}
float3[] ImageAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
return ImageAngles;
};
float3[] RelativeOffsets;
{
List<float3> RelativeOffsetList = new List<float3>();
int NSteps = (int)Math.Ceiling(offsetRange / offsetStep);
for (int z = -NSteps; z <= NSteps; z++)
for (int y = -NSteps; y <= NSteps; y++)
for (int x = -NSteps; x <= NSteps; x++)
{
float R = (float)Math.Sqrt(x * x + y * y + z * z) * offsetStep;
if (R > offsetRange + 1e-6f)
continue;
RelativeOffsetList.Add(new float3(x * offsetStep, y * offsetStep, z * offsetStep));
}
RelativeOffsets = RelativeOffsetList.ToArray();
}
float3[] HealpixAngles = Helper.GetHealpixAngles(healpixOrder, symmetry).Select(a => a * Helper.ToRad).ToArray();
float3[] ProjectionAngles = GetImageAngles(HealpixAngles);
float3[] OptimizedOrigins = new float3[NParticles];
float3[] OptimizedAngles = new float3[NParticles];
float[] BestScores = new float[NParticles].Select(v => -float.MaxValue).ToArray();
int BatchAngles = 128;
Image Projections = new Image(new int3(CoarseSize, CoarseSize, BatchAngles * NTilts), true, true);
foreach (var subset in SubsetRanges)
{
int NSubset = subset.Value.Item2 - subset.Value.Item1;
float[] ImageOffsets = new float[NSubset * NTilts * RelativeOffsets.Length * 2];
for (int o = 0; o < RelativeOffsets.Length; o++)
{
float3[] OffsetOrigins = new float3[NSubset];
for (int p = 0; p < NSubset; p++)
OffsetOrigins[p] = ParticleOrigins[subset.Value.Item1 + p] + RelativeOffsets[o];
float[] TheseOffsets = Helper.ToInterleaved(GetImageShifts(OffsetOrigins));
Array.Copy(TheseOffsets, 0, ImageOffsets, TheseOffsets.Length * o, TheseOffsets.Length);
}
int[] ShiftIDs = new int[NSubset];
int[] AngleIDs = new int[NSubset];
float[] SubsetScores = new float[NSubset];
GPU.TomoGlobalAlign(ParticleImages.GetDeviceSlice(subset.Value.Item1 * NTilts, Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDeviceSlice(subset.Value.Item1 * NTilts, Intent.Read),
ParticleWeights.GetDeviceSlice(subset.Value.Item1 * NTilts, Intent.Read),
new int2(CoarseDims),
references[subset.Key].Data.GetDevice(Intent.Read),
references[subset.Key].Data.Dims,
references[subset.Key].Oversampling,
Helper.ToInterleaved(ProjectionAngles),
(uint)HealpixAngles.Length,
ImageOffsets,
(uint)RelativeOffsets.Length,
(uint)NSubset,
(uint)NTilts,
AngleIDs,
ShiftIDs,
SubsetScores);
for (int i = 0; i < NSubset; i++)
{
OptimizedOrigins[subset.Value.Item1 + i] = ParticleOrigins[subset.Value.Item1 + i] + RelativeOffsets[ShiftIDs[i]];
OptimizedAngles[subset.Value.Item1 + i] = HealpixAngles[AngleIDs[i]];
BestScores[subset.Value.Item1 + i] = SubsetScores[i];
}
}
Projections.Dispose();
#endregion
ParticleImages?.Dispose();
ParticleCTFs?.Dispose();
ParticleWeights?.Dispose();
ShiftFactors?.Dispose();
#region Extract particles at full resolution and back-project them into the reconstruction volumes
{
GPU.SetDevice(0);
Image CTFCoords = GetCTFCoords(size, size);
int[] SortedDosePrecalc = IndicesSortedDose;
foreach (var subsetRange in SubsetRanges)
{
lock (outReconstructions[subsetRange.Key])
{
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3[] PerTiltPositions = new float3[NTilts];
float3[] PerTiltAngles = new float3[NTilts];
for (int t = 0; t < NTilts; t++)
{
PerTiltPositions[t] = OptimizedOrigins[p];
PerTiltAngles[t] = OptimizedAngles[p];
}
Image FullParticleImages = GetSubtomoImages(tiltStack, size, PerTiltPositions, true);
Image FullParticleCTFs = GetSubtomoCTFs(PerTiltPositions, CTFCoords);
FullParticleImages.Multiply(FullParticleCTFs);
FullParticleCTFs.Abs();
float3[] FullParticleAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
outReconstructions[subsetRange.Key].BackProject(FullParticleImages, FullParticleCTFs, FullParticleAngles);
FullParticleImages.Dispose();
FullParticleCTFs.Dispose();
}
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3[] PerTiltPositions = new float3[NTilts];
float3[] PerTiltAngles = new float3[NTilts];
for (int t = 0; t < NTilts; t++)
{
PerTiltPositions[t] = OptimizedOrigins[p];
PerTiltAngles[t] = OptimizedAngles[p];
}
float3[] FullParticleAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
Image FullParticleCTFs = GetSubtomoCTFs(PerTiltPositions, CTFCoords, false);
Image FullParticleCTFWeights = GetSubtomoCTFs(PerTiltPositions, CTFCoords, true);
// CTF has to be converted to complex numbers with imag = 0
float2[] CTFsComplexData = new float2[FullParticleCTFs.ElementsComplex];
float[] CTFWeightsData = new float[FullParticleCTFs.ElementsComplex];
float[] CTFsContinuousData = FullParticleCTFs.GetHostContinuousCopy();
float[] CTFWeightsContinuousData = FullParticleCTFWeights.GetHostContinuousCopy();
for (int i = 0; i < CTFsComplexData.Length; i++)
{
CTFsComplexData[i] = new float2(Math.Abs(CTFsContinuousData[i] * CTFWeightsContinuousData[i]), 0);
CTFWeightsData[i] = Math.Abs(CTFWeightsContinuousData[i]);
}
Image CTFsComplex = new Image(CTFsComplexData, FullParticleCTFs.Dims, true);
Image CTFWeights = new Image(CTFWeightsData, FullParticleCTFs.Dims, true);
outCTFReconstructions[subsetRange.Key].BackProject(CTFsComplex, CTFWeights, FullParticleAngles);
FullParticleCTFs.Dispose();
FullParticleCTFWeights.Dispose();
CTFsComplex.Dispose();
CTFWeights.Dispose();
}
outReconstructions[subsetRange.Key].FreeDevice();
outCTFReconstructions[subsetRange.Key].FreeDevice();
}
}
CTFCoords.Dispose();
}
#endregion
SaveMeta();
}
public void PerformOptimizationStep(Star tableIn, Image tiltStack, int size, int3 volumeDimensions, Dictionary<int, Projector> references, float resolution, Dictionary<int, Projector> outReconstructions, Dictionary<int, Projector> outCTFReconstructions)
{
VolumeDimensions = volumeDimensions;
#region Get rows from table
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName + "."))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return;
int NParticles = RowIndices.Count;
#endregion
#region Make sure all columns and directories are there
if (!tableIn.HasColumn("rlnImageName"))
tableIn.AddColumn("rlnImageName");
if (!tableIn.HasColumn("rlnCtfImage"))
tableIn.AddColumn("rlnCtfImage");
if (!tableIn.HasColumn("rlnParticleSelectZScore"))
tableIn.AddColumn("rlnParticleSelectZScore");
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
#endregion
#region Get subtomo positions from table
float3[] ParticleOrigins = new float3[NParticles];
float3[] ParticleOrigins2 = new float3[NParticles];
float3[] ParticleAngles = new float3[NParticles];
float3[] ParticleAngles2 = new float3[NParticles];
int[] ParticleSubset = new int[NParticles];
{
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnPosZ = tableIn.GetColumn("rlnCoordinateZ");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
string[] ColumnOriginZ = tableIn.GetColumn("rlnOriginZ");
string[] ColumnAngleRot = tableIn.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = tableIn.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = tableIn.GetColumn("rlnAnglePsi");
string[] ColumnSubset = tableIn.GetColumn("rlnRandomSubset");
string[] ColumnPosX2 = tableIn.GetColumn("rlnOriginXPrior");
string[] ColumnPosY2 = tableIn.GetColumn("rlnOriginYPrior");
string[] ColumnPosZ2 = tableIn.GetColumn("rlnOriginZPrior");
string[] ColumnAngleRot2 = tableIn.GetColumn("rlnAngleRotPrior");
string[] ColumnAngleTilt2 = tableIn.GetColumn("rlnAngleTiltPrior");
string[] ColumnAnglePsi2 = tableIn.GetColumn("rlnAnglePsiPrior");
for (int i = 0; i < NParticles; i++)
{
float3 Pos = new float3(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosZ[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Pos2 = Pos;
//if (ColumnPosX2 != null && ColumnPosY2 != null && ColumnPosZ2 != null)
// Pos2 = new float3(float.Parse(ColumnPosX2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnPosY2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnPosZ2[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Shift = new float3(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginZ[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleOrigins[i] = Pos - Shift;
ParticleOrigins2[i] = Pos2 - Shift;
float3 Angle = new float3(float.Parse(ColumnAngleRot[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAngleTilt[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnAnglePsi[RowIndices[i]], CultureInfo.InvariantCulture));
float3 Angle2 = Angle;
//if (ColumnAngleRot2 != null && ColumnAngleTilt2 != null && ColumnAnglePsi2 != null)
// Angle2 = new float3(float.Parse(ColumnAngleRot2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnAngleTilt2[RowIndices[i]], CultureInfo.InvariantCulture),
// float.Parse(ColumnAnglePsi2[RowIndices[i]], CultureInfo.InvariantCulture));
ParticleAngles[i] = Angle;
ParticleAngles2[i] = Angle2;
ParticleSubset[i] = int.Parse(ColumnSubset[RowIndices[i]]);
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", ParticleOrigins[i].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", ParticleOrigins[i].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateZ", ParticleOrigins[i].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginZ", "0.0");
}
}
#endregion
#region Deal with subsets
List<int> SubsetIDs = new List<int>();
foreach (var i in ParticleSubset)
if (!SubsetIDs.Contains(i))
SubsetIDs.Add(i);
SubsetIDs.Sort();
// For each subset, create a list of its particle IDs
Dictionary<int, List<int>> SubsetParticleIDs = SubsetIDs.ToDictionary(subsetID => subsetID, subsetID => new List<int>());
for (int i = 0; i < ParticleSubset.Length; i++)
SubsetParticleIDs[ParticleSubset[i]].Add(i);
foreach (var list in SubsetParticleIDs.Values)
list.Sort();
// Note where each subset starts and ends in a unified, sorted (by subset) particle ID list
Dictionary<int, Tuple<int, int>> SubsetRanges = new Dictionary<int, Tuple<int, int>>();
{
int Start = 0;
foreach (var pair in SubsetParticleIDs)
{
SubsetRanges.Add(pair.Key, new Tuple<int, int>(Start, Start + pair.Value.Count));
Start += pair.Value.Count;
}
}
List<int> SubsetContinuousIDs = new List<int>();
foreach (var pair in SubsetParticleIDs)
SubsetContinuousIDs.AddRange(pair.Value);
// Reorder particle information to match the order of SubsetContinuousIDs
ParticleOrigins = SubsetContinuousIDs.Select(i => ParticleOrigins[i]).ToArray();
ParticleOrigins2 = SubsetContinuousIDs.Select(i => ParticleOrigins2[i]).ToArray();
ParticleAngles = SubsetContinuousIDs.Select(i => ParticleAngles[i]).ToArray();
ParticleAngles2 = SubsetContinuousIDs.Select(i => ParticleAngles2[i]).ToArray();
ParticleSubset = SubsetContinuousIDs.Select(i => ParticleSubset[i]).ToArray();
#endregion
if (GridMovementX.Dimensions.Elements() == 1)
{
int MaxSlice = SubsetRanges.Last().Value.Item2 > 100 ? 1 : 1;
GridMovementX = new CubicGrid(new int3(MaxSlice, MaxSlice, NTilts));
GridMovementY = new CubicGrid(new int3(MaxSlice, MaxSlice, NTilts));
//GridLocalX = new CubicGrid(new int3(4, 4, 4));
//GridLocalY = new CubicGrid(new int3(4, 4, 4));
//GridLocalZ = new CubicGrid(new int3(4, 4, 4));
GridAngleX = new CubicGrid(new int3(1, 1, NTilts));
GridAngleY = new CubicGrid(new int3(1, 1, NTilts));
GridAngleZ = new CubicGrid(new int3(1, 1, NTilts));
}
if (GridLocalX.Dimensions.Elements() == 1)
{
GridLocalX = new CubicGrid(new int3(4, 4, 4));
GridLocalY = new CubicGrid(new int3(4, 4, 4));
GridLocalZ = new CubicGrid(new int3(4, 4, 4));
}
//else
//{
// GridMovementX = GridMovementX.Resize(new int3(4, 4, NTilts));
// GridMovementY = GridMovementY.Resize(new int3(4, 4, NTilts));
//}
int CoarseSize = (int)Math.Round(size * ((float)CTF.PixelSize * 2 / resolution)) / 2 * 2;
int3 CoarseDims = new int3(CoarseSize, CoarseSize, 1);
// Positions the particles were extracted at/shifted to, to calculate effectively needed shifts later
float2[] ExtractedAt = new float2[NParticles * NTilts];
// Extract images, mask and resize them, create CTFs
Image ParticleImages = new Image(new int3(CoarseSize, CoarseSize, NParticles * NTilts), true, true);
Image ParticleCTFs = new Image(new int3(CoarseSize, CoarseSize, NParticles * NTilts), true);
Image ParticleWeights = null;
Image ShiftFactors = null;
#region Preflight
float KeepBFac = GlobalBfactor;
GlobalBfactor = 0;
{
Image CTFCoords = GetCTFCoords(CoarseSize, size);
#region Precalculate vectors for shifts in Fourier space
{
float2[] ShiftFactorsData = new float2[(CoarseSize / 2 + 1) * CoarseSize];
for (int y = 0; y < CoarseSize; y++)
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x;
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
ShiftFactorsData[y * (CoarseSize / 2 + 1) + x] = new float2((float)-xx / size * 2f * (float)Math.PI,
(float)-yy / size * 2f * (float)Math.PI);
}
ShiftFactors = new Image(ShiftFactorsData, new int3(CoarseSize, CoarseSize, 1), true);
}
#endregion
#region Create mask with soft edge
Image Mask;
Image MaskSubt;
{
Image MaskBig = new Image(new int3(size, size, 1));
float MaskRadius = MainWindow.Options.ExportParticleRadius / (float)CTF.PixelSize;
float SoftEdge = 16f;
float[] MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= MaskRadius)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = (float)(Math.Cos(Math.Min(1, (R - MaskRadius) / SoftEdge) * Math.PI) * 0.5 + 0.5);
}
}
//MaskBig.WriteMRC("d_maskbig.mrc");
Mask = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
Mask.RemapToFT();
MaskBigData = MaskBig.GetHost(Intent.Write)[0];
for (int y = 0; y < size; y++)
{
int yy = y - size / 2;
yy *= yy;
for (int x = 0; x < size; x++)
{
int xx = x - size / 2;
xx *= xx;
float R = (float)Math.Sqrt(xx + yy);
if (R <= 30)
MaskBigData[y * size + x] = 1;
else
MaskBigData[y * size + x] = 0;
}
}
MaskSubt = MaskBig.AsScaled(new int2(CoarseSize, CoarseSize));
MaskSubt.RemapToFT();
MaskBig.Dispose();
}
//Mask.WriteMRC("d_masksmall.mrc");
#endregion
#region Create Fourier space mask
Image FourierMask = new Image(CoarseDims, true);
{
float[] FourierMaskData = FourierMask.GetHost(Intent.Write)[0];
int MaxR2 = CoarseSize * CoarseSize / 4;
for (int y = 0; y < CoarseSize; y++)
{
int yy = y < CoarseSize / 2 + 1 ? y : y - CoarseSize;
yy *= yy;
for (int x = 0; x < CoarseSize / 2 + 1; x++)
{
int xx = x * x;
int R2 = yy + xx;
FourierMaskData[y * (CoarseSize / 2 + 1) + x] = R2 < MaxR2 ? 1 : 0;
}
}
}
#endregion
#region For each particle, create CTFs and extract & preprocess images for entire tilt series
for (int p = 0; p < NParticles; p++)
{
float3 ParticleCoords = ParticleOrigins[p];
float3[] Positions = GetPositionInImages(ParticleCoords);
float3[] ProjAngles = GetParticleAngleInImages(ParticleCoords, ParticleAngles[p]);
Image Extracted = new Image(new int3(size, size, NTilts));
float[][] ExtractedData = Extracted.GetHost(Intent.Write);
float3[] Residuals = new float3[NTilts];
Image SubtrahendsCTF = new Image(new int3(CoarseSize, CoarseSize, NTilts), true);
// Create CTFs
{
CTFStruct[] CTFParams = new CTFStruct[NTilts];
float GridStep = 1f / (NTilts - 1);
CTFStruct[] Params = new CTFStruct[NTilts];
for (int t = 0; t < NTilts; t++)
{
decimal Defocus = (decimal)Positions[t].Z;
decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
CTF CurrCTF = CTF.GetCopy();
CurrCTF.Defocus = Defocus;
CurrCTF.DefocusDelta = DefocusDelta;
CurrCTF.DefocusAngle = DefocusAngle;
CurrCTF.Scale = (decimal)Math.Cos(Angles[t] * Helper.ToRad);
CurrCTF.Bfactor = (decimal)-Dose[t] * 8;
Params[t] = CurrCTF.ToStruct();
}
GPU.CreateCTF(ParticleCTFs.GetDeviceSlice(NTilts * p, Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CoarseDims.ElementsFFT(),
Params,
false,
(uint)NTilts);
}
//{
// CTFStruct[] CTFParams = new CTFStruct[NTilts];
// float GridStep = 1f / (NTilts - 1);
// CTFStruct[] Params = new CTFStruct[NTilts];
// for (int t = 0; t < NTilts; t++)
// {
// decimal Defocus = (decimal)Positions[t].Z;
// decimal DefocusDelta = (decimal)GridCTFDefocusDelta.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
// decimal DefocusAngle = (decimal)GridCTFDefocusAngle.GetInterpolated(new float3(0.5f, 0.5f, t * GridStep));
// CTF CurrCTF = CTF.GetCopy();
// CurrCTF.Defocus = Defocus;
// CurrCTF.DefocusDelta = DefocusDelta;
// CurrCTF.DefocusAngle = DefocusAngle;
// CurrCTF.Scale = 1;
// CurrCTF.Bfactor = 0;
// Params[t] = CurrCTF.ToStruct();
// }
// GPU.CreateCTF(SubtrahendsCTF.GetDevice(Intent.Write),
// CTFCoords.GetDevice(Intent.Read),
// (uint)CoarseDims.ElementsFFT(),
// Params,
// false,
// (uint)NTilts);
//}
// Extract images
{
for (int t = 0; t < NTilts; t++)
{
ExtractedAt[p * NTilts + t] = new float2(Positions[t].X, Positions[t].Y);
Positions[t] -= size / 2;
int2 IntPosition = new int2((int)Positions[t].X, (int)Positions[t].Y);
float2 Residual = new float2(-(Positions[t].X - IntPosition.X), -(Positions[t].Y - IntPosition.Y));
Residuals[t] = new float3(Residual / size * CoarseSize);
float[] OriginalData;
lock (tiltStack)
OriginalData = tiltStack.GetHost(Intent.Read)[t];
float[] ImageData = ExtractedData[t];
for (int y = 0; y < size; y++)
{
int PosY = (y + IntPosition.Y + tiltStack.Dims.Y) % tiltStack.Dims.Y;
for (int x = 0; x < size; x++)
{
int PosX = (x + IntPosition.X + tiltStack.Dims.X) % tiltStack.Dims.X;
ImageData[y * size + x] = OriginalData[PosY * tiltStack.Dims.X + PosX];
}
}
}
GPU.NormParticles(Extracted.GetDevice(Intent.Read),
Extracted.GetDevice(Intent.Write),
new int3(size, size, 1),
(uint)(MainWindow.Options.ExportParticleRadius / CTF.PixelSize),
true,
(uint)NTilts);
Image Scaled = Extracted.AsScaled(new int2(CoarseSize, CoarseSize));
//Scaled.WriteMRC("d_scaled.mrc");
Extracted.Dispose();
Scaled.ShiftSlices(Residuals);
Scaled.RemapToFT();
//GPU.NormalizeMasked(Scaled.GetDevice(Intent.Read),
// Scaled.GetDevice(Intent.Write),
// MaskSubt.GetDevice(Intent.Read),
// (uint)Scaled.ElementsSliceReal,
// (uint)NTilts);
//{
// //Image SubtrahendsFT = subtrahendReference.Project(new int2(CoarseSize, CoarseSize), ProjAngles, CoarseSize / 2);
// //SubtrahendsFT.Multiply(SubtrahendsCTF);
// //Image Subtrahends = SubtrahendsFT.AsIFFT();
// //SubtrahendsFT.Dispose();
// ////GPU.NormalizeMasked(Subtrahends.GetDevice(Intent.Read),
// //// Subtrahends.GetDevice(Intent.Write),
// //// MaskSubt.GetDevice(Intent.Read),
// //// (uint)Subtrahends.ElementsSliceReal,
// //// (uint)NTilts);
// //Scaled.Subtract(Subtrahends);
// //Subtrahends.Dispose();
// Image FocusMaskFT = maskReference.Project(new int2(CoarseSize, CoarseSize), ProjAngles, CoarseSize / 2);
// Image FocusMask = FocusMaskFT.AsIFFT();
// FocusMaskFT.Dispose();
// Scaled.Multiply(FocusMask);
// FocusMask.Dispose();
//}
Scaled.MultiplySlices(Mask);
GPU.FFT(Scaled.GetDevice(Intent.Read),
ParticleImages.GetDeviceSlice(p * NTilts, Intent.Write),
CoarseDims,
(uint)NTilts);
Scaled.Dispose();
SubtrahendsCTF.Dispose();
}
}
#endregion
ParticleCTFs.MultiplySlices(FourierMask);
Mask.Dispose();
FourierMask.Dispose();
MaskSubt.Dispose();
Image ParticleCTFsAbs = new Image(ParticleCTFs.GetDevice(Intent.Read), ParticleCTFs.Dims, true);
ParticleCTFsAbs.Abs();
ParticleWeights = ParticleCTFsAbs.AsSum2D();
ParticleCTFsAbs.Dispose();
{
float[] ParticleWeightsData = ParticleWeights.GetHost(Intent.ReadWrite)[0];
float Max = MathHelper.Max(ParticleWeightsData);
for (int i = 0; i < ParticleWeightsData.Length; i++)
ParticleWeightsData[i] /= Max;
}
CTFCoords.Dispose();
//Image CheckImages = ParticleImages.AsIFFT();
//CheckImages.WriteMRC("d_particleimages.mrc");
//CheckImages.Dispose();
//ParticleCTFs.WriteMRC("d_particlectfs.mrc");
}
GlobalBfactor = KeepBFac;
#endregion
bool DoPerParticleMotion = true;
bool DoImageAlignment = true;
#region BFGS evaluation and gradient
double[] StartParams;
Func<double[], Tuple<float2[], float3[]>> GetImageShiftsAndAngles;
Func<double[], float2[]> GetImageShifts;
Func<float3[], Image> GetProjections;
Func<double[], double[]> EvalIndividual;
Func<double[], double> Eval;
Func<double[], double[]> Gradient;
{
List<double> StartParamsList = new List<double>();
StartParamsList.AddRange(CreateVectorFromGrids(Dimensions.X));
StartParamsList.AddRange(CreateVectorFromParameters(ParticleOrigins, ParticleOrigins2, ParticleAngles, ParticleAngles2, size));
StartParams = StartParamsList.ToArray();
// Remember where the values for each grid are stored in the optimized vector
List<Tuple<int, int>> VectorGridRanges = new List<Tuple<int, int>>();
List<int> GridElements = new List<int>();
List<int> GridSliceElements = new List<int>();
{
int Start = 0;
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridMovementX.Dimensions.Elements()));
Start += (int)GridMovementX.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridMovementY.Dimensions.Elements()));
Start += (int)GridMovementY.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridAngleX.Dimensions.Elements()));
Start += (int)GridAngleX.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridAngleY.Dimensions.Elements()));
Start += (int)GridAngleY.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridAngleZ.Dimensions.Elements()));
Start += (int)GridAngleZ.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridLocalX.Dimensions.Elements()));
Start += (int)GridLocalX.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridLocalY.Dimensions.Elements()));
Start += (int)GridLocalY.Dimensions.Elements();
VectorGridRanges.Add(new Tuple<int, int>(Start, Start + (int)GridLocalZ.Dimensions.Elements()));
GridElements.Add((int)GridMovementX.Dimensions.Elements());
GridElements.Add((int)GridMovementY.Dimensions.Elements());
GridElements.Add((int)GridAngleX.Dimensions.Elements());
GridElements.Add((int)GridAngleY.Dimensions.Elements());
GridElements.Add((int)GridAngleZ.Dimensions.Elements());
GridElements.Add((int)GridLocalX.Dimensions.Elements());
GridElements.Add((int)GridLocalY.Dimensions.Elements());
GridElements.Add((int)GridLocalZ.Dimensions.Elements());
GridSliceElements.Add((int)GridMovementX.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridMovementY.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridAngleX.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridAngleY.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridAngleZ.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridLocalX.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridLocalY.Dimensions.ElementsSlice());
GridSliceElements.Add((int)GridLocalZ.Dimensions.ElementsSlice());
}
int NVectorGridParams = VectorGridRanges.Last().Item2;
int NVectorParticleParams = NParticles * 12;
GetImageShiftsAndAngles = input =>
{
// Retrieve particle positions & angles, and grids from input vector
float3[] NewPositions, NewPositions2, NewAngles, NewAngles2;
GetParametersFromVector(input, NParticles, size, out NewPositions, out NewPositions2, out NewAngles, out NewAngles2);
SetGridsFromVector(input, Dimensions.X);
// Using current positions, angles and grids, get parameters for image shifts and reference projection angles
float2[] ImageShifts = new float2[NParticles * NTilts];
float3[] ImageAngles = new float3[NParticles * NTilts];
float3[] PerTiltPositions = new float3[NParticles * NTilts];
float3[] PerTiltAngles = new float3[NParticles * NTilts];
int[] SortedDosePrecalc = IndicesSortedDose;
for (int p = 0; p < NParticles; p++)
{
if (DoPerParticleMotion)
{
float3 CoordsDiff = NewPositions2[p] - NewPositions[p];
float3 AnglesDiff = NewAngles2[p] - NewAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[p * NTilts + t] = NewPositions[p] + CoordsDiff * DoseID;
PerTiltAngles[p * NTilts + t] = NewAngles[p] + AnglesDiff * DoseID;
}
}
else
{
for (int t = 0; t < NTilts; t++)
{
PerTiltPositions[p * NTilts + t] = NewPositions[p];
PerTiltAngles[p * NTilts + t] = NewAngles[p];
}
}
}
float3[] CurrPositions = GetPositionInImages(PerTiltPositions);
float3[] CurrAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
for (int i = 0; i < ImageShifts.Length; i++)
{
ImageShifts[i] = new float2(ExtractedAt[i].X - CurrPositions[i].X,
ExtractedAt[i].Y - CurrPositions[i].Y); // -diff because those are extraction positions, i. e. opposite direction of shifts
ImageAngles[i] = CurrAngles[i];
}
return new Tuple<float2[], float3[]>(ImageShifts, ImageAngles);
};
GetImageShifts = input =>
{
// Retrieve particle positions & angles, and grids from input vector
float3[] NewPositions, NewPositions2, NewAngles, NewAngles2;
GetParametersFromVector(input, NParticles, size, out NewPositions, out NewPositions2, out NewAngles, out NewAngles2);
SetGridsFromVector(input, Dimensions.X);
// Using current positions, angles and grids, get parameters for image shifts and reference projection angles
float2[] ImageShifts = new float2[NParticles * NTilts];
float3[] PerTiltPositions = new float3[NParticles * NTilts];
int[] SortedDosePrecalc = IndicesSortedDose;
for (int p = 0; p < NParticles; p++)
{
if (DoPerParticleMotion)
{
float3 CoordsDiff = NewPositions2[p] - NewPositions[p];
float3 AnglesDiff = NewAngles2[p] - NewAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[p * NTilts + t] = NewPositions[p] + CoordsDiff * DoseID;
}
}
else
{
for (int t = 0; t < NTilts; t++)
PerTiltPositions[p * NTilts + t] = NewPositions[p];
}
}
float3[] CurrPositions = GetPositionInImages(PerTiltPositions);
for (int i = 0; i < ImageShifts.Length; i++)
ImageShifts[i] = new float2(ExtractedAt[i].X - CurrPositions[i].X,
ExtractedAt[i].Y - CurrPositions[i].Y); // -diff because those are extraction positions, i. e. opposite direction of shifts
return ImageShifts;
};
GetProjections = imageAngles =>
{
Image Projections = new Image(IntPtr.Zero, new int3(CoarseSize, CoarseSize, NParticles * NTilts), true, true);
foreach (var subset in SubsetRanges)
{
Projector Reference = references[subset.Key];
int SubsetStart = subset.Value.Item1 * NTilts;
int SubsetEnd = subset.Value.Item2 * NTilts;
float3[] SubsetAngles = imageAngles.Skip(SubsetStart).Take(SubsetEnd - SubsetStart).ToArray();
GPU.ProjectForward(Reference.Data.GetDevice(Intent.Read),
Projections.GetDeviceSlice(SubsetStart, Intent.Write),
Reference.Data.Dims,
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(SubsetAngles),
Reference.Oversampling,
(uint)(SubsetEnd - SubsetStart));
}
/*Image CheckProjections = Projections.AsIFFT();
//CheckProjections.RemapFromFT();
CheckProjections.WriteMRC("d_projections.mrc");
CheckProjections.Dispose();*/
return Projections;
};
EvalIndividual = input =>
{
Tuple<float2[], float3[]> ShiftsAndAngles = GetImageShiftsAndAngles(input);
Image Projections = GetProjections(ShiftsAndAngles.Item2);
float[] Results = new float[NParticles * NTilts];
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsAndAngles.Item1),
Results,
(uint)(NParticles * NTilts));
Projections.Dispose();
return Results.Select(i => (double)i).ToArray();
};
int OptimizationIterations = 0;
bool GetOut = false;
double Delta = 0.1;
float Delta2 = 2 * (float)Delta;
int[] WarpGridIDs = { 5, 6, 7 };
Dictionary<int, float2[][]> WiggleWeightsWarp = new Dictionary<int, float2[][]>();
foreach (var gridID in WarpGridIDs)
{
int NElements = GridElements[gridID];
WiggleWeightsWarp.Add(gridID, new float2[NElements][]);
for (int ge = 0; ge < NElements; ge++)
{
double[] InputMinus = new double[StartParams.Length], InputPlus = new double[StartParams.Length];
for (int i = 0; i < StartParams.Length; i++)
{
InputMinus[i] = StartParams[i];
InputPlus[i] = StartParams[i];
}
InputMinus[VectorGridRanges[gridID].Item1 + ge] -= Delta;
InputPlus[VectorGridRanges[gridID].Item1 + ge] += Delta;
float2[] ImageShiftsPlus = GetImageShifts(InputPlus);
float2[] ImageShiftsMinus = GetImageShifts(InputMinus);
float2[] Weights = new float2[ImageShiftsPlus.Length];
for (int i = 0; i < ImageShiftsPlus.Length; i++)
Weights[i] = (ImageShiftsPlus[i] - ImageShiftsMinus[i]) / Delta2;
WiggleWeightsWarp[gridID][ge] = Weights;
}
}
Eval = input =>
{
double Result = EvalIndividual(input).Sum();
lock (tableIn)
Debug.WriteLine(GPU.GetDevice() + ", " + RootName + ": " + Result);
OptimizationIterations++;
return Result;
};
Func<double[], double[], double, double[]> GradientParticles = (inputMinus, inputPlus, delta) =>
{
double[] EvalMinus = EvalIndividual(inputMinus);
double[] EvalPlus = EvalIndividual(inputPlus);
double[] Diff = new double[EvalMinus.Length];
for (int i = 0; i < Diff.Length; i++)
Diff[i] = (EvalPlus[i] - EvalMinus[i]) / (2 * delta);
return Diff;
};
Gradient = input =>
{
double[] Result = new double[input.Length];
if (OptimizationIterations > 60)
return Result;
float2[] ImageShiftGradients = new float2[NParticles * NTilts];
#region Compute gradient for individual image shifts
{
Tuple<float2[], float3[]> ShiftsAndAngles = GetImageShiftsAndAngles(input);
Image Projections = GetProjections(ShiftsAndAngles.Item2);
float2[] ShiftsXPlus = new float2[NParticles * NTilts];
float2[] ShiftsXMinus = new float2[NParticles * NTilts];
float2[] ShiftsYPlus = new float2[NParticles * NTilts];
float2[] ShiftsYMinus = new float2[NParticles * NTilts];
float2 DeltaX = new float2((float)Delta, 0);
float2 DeltaY = new float2(0, (float)Delta);
for (int i = 0; i < ShiftsXPlus.Length; i++)
{
ShiftsXPlus[i] = ShiftsAndAngles.Item1[i] + DeltaX;
ShiftsXMinus[i] = ShiftsAndAngles.Item1[i] - DeltaX;
ShiftsYPlus[i] = ShiftsAndAngles.Item1[i] + DeltaY;
ShiftsYMinus[i] = ShiftsAndAngles.Item1[i] - DeltaY;
}
float[] ScoresXPlus = new float[NParticles * NTilts];
float[] ScoresXMinus = new float[NParticles * NTilts];
float[] ScoresYPlus = new float[NParticles * NTilts];
float[] ScoresYMinus = new float[NParticles * NTilts];
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsXPlus),
ScoresXPlus,
(uint)(NParticles * NTilts));
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsXMinus),
ScoresXMinus,
(uint)(NParticles * NTilts));
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsYPlus),
ScoresYPlus,
(uint)(NParticles * NTilts));
GPU.TomoRefineGetDiff(ParticleImages.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
ParticleCTFs.GetDevice(Intent.Read),
ParticleWeights.GetDevice(Intent.Read),
new int2(CoarseSize, CoarseSize),
Helper.ToInterleaved(ShiftsYMinus),
ScoresYMinus,
(uint)(NParticles * NTilts));
Projections.Dispose();
for (int i = 0; i < ImageShiftGradients.Length; i++)
{
ImageShiftGradients[i] = new float2((ScoresXPlus[i] - ScoresXMinus[i]) / Delta2,
(ScoresYPlus[i] - ScoresYMinus[i]) / Delta2);
}
}
#endregion
// First, do particle parameters, i. e. 3D position within tomogram, rotation, across 2 points in time
// Altering each particle's parameters results in a change in its NTilts images, but nothing else
{
int[] TranslationIDs = DoPerParticleMotion ? new[] { 0, 1, 2, 3, 4, 5 } : new[] { 0, 1, 2 };
int[] RotationIDs = DoPerParticleMotion ? new [] {6, 7, 8, 9, 10, 11} : new [] { 6, 7, 8};
foreach (var paramID in RotationIDs)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int p = 0; p < NParticles; p++)
{
InputMinus[NVectorGridParams + p * 12 + paramID] -= Delta;
InputPlus[NVectorGridParams + p * 12 + paramID] += Delta;
}
double[] ResultParticles = GradientParticles(InputMinus, InputPlus, Delta);
for (int p = 0; p < NParticles; p++)
{
double ParticleSum = 0;
for (int t = 0; t < NTilts; t++)
ParticleSum += ResultParticles[p * NTilts + t];
Result[NVectorGridParams + p * 12 + paramID] = ParticleSum;
}
}
// Translation-related gradients can all be computed efficiently from previously retrieved per-image gradients
foreach (var paramID in TranslationIDs)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int p = 0; p < NParticles; p++)
{
InputMinus[NVectorGridParams + p * 12 + paramID] -= Delta;
InputPlus[NVectorGridParams + p * 12 + paramID] += Delta;
}
float2[] ImageShiftsPlus = GetImageShifts(InputPlus);
float2[] ImageShiftsMinus = GetImageShifts(InputMinus);
for (int p = 0; p < NParticles; p++)
{
double ParticleSum = 0;
for (int t = 0; t < NTilts; t++)
{
int i = p * NTilts + t;
float2 ShiftDelta = (ImageShiftsPlus[i] - ImageShiftsMinus[i]) / Delta2;
float ShiftGradient = ShiftDelta.X * ImageShiftGradients[i].X + ShiftDelta.Y * ImageShiftGradients[i].Y;
ParticleSum += ShiftGradient;
}
Result[NVectorGridParams + p * 12 + paramID] = ParticleSum;
}
}
// If there is no per-particle motion, just copy the gradients for these parameters from parameterIDs 0-5
if (!DoPerParticleMotion)
{
int[] RedundantIDs = { 3, 4, 5, 9, 10, 11 };
foreach (var paramID in RedundantIDs)
for (int p = 0; p < NParticles; p++)
Result[NVectorGridParams + p * 12 + paramID] = Result[NVectorGridParams + p * 12 + paramID - 3];
}
}
// Now deal with grids. Each grid slice (i. e. temporal point) will correspond to one tilt only, thus the gradient
// for each slice is the (weighted, in case of spatial resolution) sum of NParticles images in the corresponding tilt.
if (DoImageAlignment)
{
int[] RotationGridIDs = { 2, 3, 4 };
foreach (var gridID in RotationGridIDs)
{
int SliceElements = GridSliceElements[gridID];
for (int se = 0; se < SliceElements; se++)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int gp = VectorGridRanges[gridID].Item1 + se; gp < VectorGridRanges[gridID].Item2; gp += SliceElements)
{
InputMinus[gp] -= Delta;
InputPlus[gp] += Delta;
}
double[] ResultParticles = GradientParticles(InputMinus, InputPlus, Delta);
for (int i = 0; i < ResultParticles.Length; i++)
{
int GridTime = i % NTilts;
Result[VectorGridRanges[gridID].Item1 + GridTime * SliceElements + se] += ResultParticles[i];
}
}
}
// Translation-related gradients can all be computed efficiently from previously retrieved per-image gradients
int[] TranslationGridIDs = { 0, 1 };
foreach (var gridID in TranslationGridIDs)
{
int SliceElements = GridSliceElements[gridID];
for (int se = 0; se < SliceElements; se++)
{
double[] InputMinus = new double[input.Length], InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
InputMinus[i] = input[i];
InputPlus[i] = input[i];
}
for (int gp = VectorGridRanges[gridID].Item1 + se; gp < VectorGridRanges[gridID].Item2; gp += SliceElements)
{
InputMinus[gp] -= Delta;
InputPlus[gp] += Delta;
}
float2[] ImageShiftsPlus = GetImageShifts(InputPlus);
float2[] ImageShiftsMinus = GetImageShifts(InputMinus);
for (int i = 0; i < ImageShiftsPlus.Length; i++)
{
float2 ShiftDelta = (ImageShiftsPlus[i] - ImageShiftsMinus[i]) / Delta2;
float ShiftGradient = ShiftDelta.X * ImageShiftGradients[i].X + ShiftDelta.Y * ImageShiftGradients[i].Y;
int GridSlice = i % NTilts;
Result[VectorGridRanges[gridID].Item1 + GridSlice * SliceElements + se] += ShiftGradient;
}
}
}
// Warp grids don't have any shortcuts for getting multiple gradients at once, so they use pre-calculated wiggle weights
foreach (var gridID in WarpGridIDs)
{
int NElements = GridElements[gridID];
for (int ge = 0; ge < NElements; ge++)
{
float2[] Weights = WiggleWeightsWarp[gridID][ge];
for (int i = 0; i < Weights.Length; i++)
{
float2 ShiftDelta = Weights[i];
float ShiftGradient = ShiftDelta.X * ImageShiftGradients[i].X + ShiftDelta.Y * ImageShiftGradients[i].Y;
Result[VectorGridRanges[gridID].Item1 + ge] += ShiftGradient;
}
}
}
}
return Result;
};
}
#endregion
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Gradient);
Optimizer.Epsilon = 3e-7;
Optimizer.Maximize(StartParams);
float3[] OptimizedOrigins, OptimizedOrigins2, OptimizedAngles, OptimizedAngles2;
GetParametersFromVector(StartParams, NParticles, size, out OptimizedOrigins, out OptimizedOrigins2, out OptimizedAngles, out OptimizedAngles2);
SetGridsFromVector(StartParams, Dimensions.X);
#region Calculate correlation scores, update table with new positions and angles
{
double[] ImageScores = EvalIndividual(StartParams);
float[] ParticleScores = new float[NParticles];
for (int i = 0; i < ImageScores.Length; i++)
ParticleScores[i / NTilts] += (float)ImageScores[i];
//if (!tableIn.HasColumn("rlnOriginXPrior"))
// tableIn.AddColumn("rlnOriginXPrior");
//if (!tableIn.HasColumn("rlnOriginYPrior"))
// tableIn.AddColumn("rlnOriginYPrior");
//if (!tableIn.HasColumn("rlnOriginZPrior"))
// tableIn.AddColumn("rlnOriginZPrior");
//if (!tableIn.HasColumn("rlnAngleRotPrior"))
// tableIn.AddColumn("rlnAngleRotPrior");
//if (!tableIn.HasColumn("rlnAngleTiltPrior"))
// tableIn.AddColumn("rlnAngleTiltPrior");
//if (!tableIn.HasColumn("rlnAnglePsiPrior"))
// tableIn.AddColumn("rlnAnglePsiPrior");
lock (tableIn)
for (int p = 0; p < NParticles; p++)
{
int Row = RowIndices[SubsetContinuousIDs[p]];
tableIn.SetRowValue(Row, "rlnCoordinateX", OptimizedOrigins[p].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnCoordinateY", OptimizedOrigins[p].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnCoordinateZ", OptimizedOrigins[p].Z.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnOriginXPrior", OptimizedOrigins2[p].X.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnOriginYPrior", OptimizedOrigins2[p].Y.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnOriginZPrior", OptimizedOrigins2[p].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnAngleRot", OptimizedAngles[p].X.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnAngleTilt", OptimizedAngles[p].Y.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnAnglePsi", OptimizedAngles[p].Z.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnAngleRotPrior", OptimizedAngles2[p].X.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnAngleTiltPrior", OptimizedAngles2[p].Y.ToString(CultureInfo.InvariantCulture));
//tableIn.SetRowValue(Row, "rlnAnglePsiPrior", OptimizedAngles2[p].Z.ToString(CultureInfo.InvariantCulture));
tableIn.SetRowValue(Row, "rlnParticleSelectZScore", ParticleScores[p].ToString(CultureInfo.InvariantCulture));
}
}
#endregion
ParticleImages?.Dispose();
ParticleCTFs?.Dispose();
ParticleWeights?.Dispose();
ShiftFactors?.Dispose();
#region Extract particles at full resolution and back-project them into the reconstruction volumes
{
GPU.SetDevice(0);
Image CTFCoords = GetCTFCoords(size, size);
int[] SortedDosePrecalc = IndicesSortedDose;
foreach (var subsetRange in SubsetRanges)
{
lock (outReconstructions[subsetRange.Key])
{
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3[] PerTiltPositions = new float3[NTilts];
float3[] PerTiltAngles = new float3[NTilts];
float3 CoordsDiff = OptimizedOrigins2[p] - OptimizedOrigins[p];
float3 AnglesDiff = OptimizedAngles2[p] - OptimizedAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[t] = OptimizedOrigins[p] + CoordsDiff * DoseID;
PerTiltAngles[t] = OptimizedAngles[p] + AnglesDiff * DoseID;
}
Image FullParticleImages = GetSubtomoImages(tiltStack, size, PerTiltPositions, true);
Image FullParticleCTFs = GetSubtomoCTFs(PerTiltPositions, CTFCoords);
FullParticleImages.Multiply(FullParticleCTFs);
FullParticleCTFs.Abs();
float3[] FullParticleAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
outReconstructions[subsetRange.Key].BackProject(FullParticleImages, FullParticleCTFs, FullParticleAngles);
FullParticleImages.Dispose();
FullParticleCTFs.Dispose();
}
for (int p = subsetRange.Value.Item1; p < subsetRange.Value.Item2; p++)
{
float3[] PerTiltPositions = new float3[NTilts];
float3[] PerTiltAngles = new float3[NTilts];
float3 CoordsDiff = OptimizedOrigins2[p] - OptimizedOrigins[p];
float3 AnglesDiff = OptimizedAngles2[p] - OptimizedAngles[p];
for (int t = 0; t < NTilts; t++)
{
float DoseID = SortedDosePrecalc[t] / (float)(NTilts - 1);
PerTiltPositions[t] = OptimizedOrigins[p] + CoordsDiff * DoseID;
PerTiltAngles[t] = OptimizedAngles[p] + AnglesDiff * DoseID;
}
float3[] FullParticleAngles = GetParticleAngleInImages(PerTiltPositions, PerTiltAngles);
Image FullParticleCTFs = GetSubtomoCTFs(PerTiltPositions, CTFCoords, false);
Image FullParticleCTFWeights = GetSubtomoCTFs(PerTiltPositions, CTFCoords, true);
// CTF has to be converted to complex numbers with imag = 0
float2[] CTFsComplexData = new float2[FullParticleCTFs.ElementsComplex];
float[] CTFWeightsData = new float[FullParticleCTFs.ElementsComplex];
float[] CTFsContinuousData = FullParticleCTFs.GetHostContinuousCopy();
float[] CTFWeightsContinuousData = FullParticleCTFWeights.GetHostContinuousCopy();
for (int i = 0; i < CTFsComplexData.Length; i++)
{
CTFsComplexData[i] = new float2(Math.Abs(CTFsContinuousData[i] * CTFWeightsContinuousData[i]), 0);
CTFWeightsData[i] = Math.Abs(CTFWeightsContinuousData[i]);
}
Image CTFsComplex = new Image(CTFsComplexData, FullParticleCTFs.Dims, true);
Image CTFWeights = new Image(CTFWeightsData, FullParticleCTFs.Dims, true);
outCTFReconstructions[subsetRange.Key].BackProject(CTFsComplex, CTFWeights, FullParticleAngles);
FullParticleCTFs.Dispose();
FullParticleCTFWeights.Dispose();
CTFsComplex.Dispose();
CTFWeights.Dispose();
}
outReconstructions[subsetRange.Key].FreeDevice();
outCTFReconstructions[subsetRange.Key].FreeDevice();
}
}
CTFCoords.Dispose();
}
#endregion
SaveMeta();
}
public void PerformComparison(MapHeader originalHeader, Star stardata, Image refft, Image maskft, decimal scaleFactor)
{
int NFrames = originalHeader.Dimensions.Z;
int2 DimsImage = new int2(originalHeader.Dimensions);
float3[] PositionsGrid;
float3[] PositionsShift;
float3[] ParticleAngles;
List<int> RowIndices = new List<int>();
{
string[] ColumnNames = stardata.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnNames.Length; i++)
if (ColumnNames[i].Contains(RootName))
RowIndices.Add(i);
string[] ColumnOriginX = stardata.GetColumn("rlnCoordinateX");
string[] ColumnOriginY = stardata.GetColumn("rlnCoordinateY");
string[] ColumnShiftX = stardata.GetColumn("rlnOriginX");
string[] ColumnShiftY = stardata.GetColumn("rlnOriginY");
string[] ColumnAngleRot = stardata.GetColumn("rlnAngleRot");
string[] ColumnAngleTilt = stardata.GetColumn("rlnAngleTilt");
string[] ColumnAnglePsi = stardata.GetColumn("rlnAnglePsi");
PositionsGrid = new float3[RowIndices.Count];
PositionsShift = new float3[RowIndices.Count];
ParticleAngles = new float3[RowIndices.Count];
{
int i = 0;
foreach (var nameIndex in RowIndices)
{
float OriginX = float.Parse(ColumnOriginX[nameIndex]);
float OriginY = float.Parse(ColumnOriginY[nameIndex]);
float ShiftX = float.Parse(ColumnShiftX[nameIndex]);
float ShiftY = float.Parse(ColumnShiftY[nameIndex]);
PositionsGrid[i] = new float3((OriginX - ShiftX) / DimsImage.X, (OriginY - ShiftY) / DimsImage.Y, 0);
PositionsShift[i] = new float3(ShiftX, ShiftY, 0f);
ParticleAngles[i] = new float3(-float.Parse(ColumnAngleRot[nameIndex]) * Helper.ToRad,
-float.Parse(ColumnAngleTilt[nameIndex]) * Helper.ToRad,
-float.Parse(ColumnAnglePsi[nameIndex]) * Helper.ToRad);
i++;
}
}
}
int NPositions = PositionsGrid.Length;
if (NPositions == 0)
return;
Image Particles = StageDataLoad.LoadMap(ParticlesPath, new int2(1, 1), 0, typeof (float));
int2 DimsRegion = new int2(Particles.Dims.X, Particles.Dims.X);
Particles.ShiftSlices(PositionsShift);
int MinFreqInclusive = (int)(MainWindow.Options.MovementRangeMin * DimsRegion.X / 2);
int MaxFreqExclusive = (int)(MainWindow.Options.MovementRangeMax * DimsRegion.X / 2);
int NFreq = MaxFreqExclusive - MinFreqInclusive;
Image ParticleMasksFT = maskft.AsProjections(ParticleAngles, DimsRegion, MainWindow.Options.ProjectionOversample);
Image ParticleMasks = ParticleMasksFT.AsIFFT();
ParticleMasksFT.Dispose();
ParticleMasks.RemapFromFT();
Parallel.ForEach(ParticleMasks.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(25f, slice[i])) - 2) / 23f;
});
Image ProjectionsFT = refft.AsProjections(ParticleAngles, DimsRegion, MainWindow.Options.ProjectionOversample);
Image Projections = ProjectionsFT.AsIFFT();
ProjectionsFT.Dispose();
// Addresses for CTF simulation
Image CTFCoordsCart = new Image(new int3(DimsRegion), true, true);
{
float2[] CoordsData = new float2[CTFCoordsCart.ElementsSliceComplex];
Helper.ForEachElementFT(DimsRegion, (x, y, xx, yy, r, a) => CoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r / DimsRegion.X, a));
CTFCoordsCart.UpdateHostWithComplex(new[] { CoordsData });
CTFCoordsCart.RemapToFT();
}
float[] ValuesDefocus = GridCTF.GetInterpolatedNative(PositionsGrid);
CTFStruct[] PositionsCTF = ValuesDefocus.Select(v =>
{
CTF Altered = CTF.GetCopy();
Altered.Defocus = (decimal)v;
//Altered.Bfactor = -MainWindow.Options.MovementBfactor;
return Altered.ToStruct();
}).ToArray();
Image Scores = new Image(IntPtr.Zero, new int3(NPositions, 1, 1));
GPU.CompareParticles(Particles.GetDevice(Intent.Read),
ParticleMasks.GetDevice(Intent.Read),
Projections.GetDevice(Intent.Read),
DimsRegion,
CTFCoordsCart.GetDevice(Intent.Read),
PositionsCTF,
MinFreqInclusive,
MaxFreqExclusive,
Scores.GetDevice(Intent.Write),
(uint)NPositions);
float[] ScoresData = Scores.GetHost(Intent.Read)[0];
for (int p = 0; p < NPositions; p++)
{
stardata.SetRowValue(RowIndices[p], "rlnCtfFigureOfMerit", ScoresData[p].ToString(CultureInfo.InvariantCulture));
}
Scores.Dispose();
Projections.Dispose();
ParticleMasks.Dispose();
CTFCoordsCart.Dispose();
Particles.Dispose();
}
public void ExportParticlesMovieOld(Star table, int size)
{
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = table.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return;
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
MapHeader OriginalHeader = MapHeader.ReadFromFile(Path,
new int2(MainWindow.Options.InputDatWidth, MainWindow.Options.InputDatHeight),
MainWindow.Options.InputDatOffset,
ImageFormatsHelper.StringToType(MainWindow.Options.InputDatType));
/*Image OriginalStack = StageDataLoad.LoadMap(Path,
new int2(MainWindow.Options.InputDatWidth, MainWindow.Options.InputDatHeight),
MainWindow.Options.InputDatOffset,
ImageFormatsHelper.StringToType(MainWindow.Options.InputDatType));*/
//OriginalStack.Xray(20f);
int3 Dims = OriginalHeader.Dimensions;
int3 DimsRegion = new int3(size, size, 1);
int NParticles = RowIndices.Count / Dimensions.Z;
float PixelSize = (float)(MainWindow.Options.CTFPixelMin + MainWindow.Options.CTFPixelMax) * 0.5f;
float PixelDelta = (float)(MainWindow.Options.CTFPixelMax - MainWindow.Options.CTFPixelMin) * 0.5f;
float PixelAngle = (float)MainWindow.Options.CTFPixelAngle / (float)(180.0 / Math.PI);
Image CTFCoords;
{
float2[] CTFCoordsData = new float2[DimsRegion.ElementsSlice()];
//Helper.ForEachElementFT(new int2(DimsRegion), (x, y, xx, yy) =>
for (int y = 0; y < DimsRegion.Y; y++)
for (int x = 0; x < DimsRegion.X / 2 + 1; x++)
{
int xx = x;
int yy = y < DimsRegion.Y / 2 + 1 ? y : y - DimsRegion.Y;
float xs = xx / (float)DimsRegion.X;
float ys = yy / (float)DimsRegion.Y;
float r = (float)Math.Sqrt(xs * xs + ys * ys);
float angle = (float)(Math.Atan2(yy, xx));
float CurrentPixelSize = PixelSize + PixelDelta * (float)Math.Cos(2f * (angle - PixelAngle));
CTFCoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r / CurrentPixelSize, angle);
} //);
CTFCoords = new Image(CTFCoordsData, DimsRegion.Slice(), true);
//CTFCoords.RemapToFT();
}
Image CTFFreq = CTFCoords.AsReal();
Image CTFStack = new Image(new int3(size, size, NParticles * Dimensions.Z), true);
int CTFStackIndex = 0;
string[] ColumnPosX = table.GetColumn("rlnCoordinateX");
string[] ColumnPosY = table.GetColumn("rlnCoordinateY");
int3[] Origins = new int3[NParticles];
for (int i = 0; i < NParticles; i++)
Origins[i] = new int3((int)double.Parse(ColumnPosX[RowIndices[i]]) - DimsRegion.X * 2 / 2,
(int)double.Parse(ColumnPosY[RowIndices[i]]) - DimsRegion.Y * 2 / 2,
0);
int IndexOffset = RowIndices[0];
string[] ColumnOriginX = table.GetColumn("rlnOriginX");
string[] ColumnOriginY = table.GetColumn("rlnOriginY");
string[] ColumnPriorX = table.GetColumn("rlnOriginXPrior");
string[] ColumnPriorY = table.GetColumn("rlnOriginYPrior");
float2[] ShiftPriors = new float2[NParticles];
for (int i = 0; i < NParticles; i++)
ShiftPriors[i] = new float2(float.Parse(ColumnPriorX[IndexOffset + i]),
float.Parse(ColumnPriorY[IndexOffset + i]));
float2[][] ParticleTracks = new float2[NParticles][];
for (int i = 0; i < NParticles; i++)
{
ParticleTracks[i] = new float2[Dimensions.Z];
for (int z = 0; z < Dimensions.Z; z++)
ParticleTracks[i][z] = new float2(float.Parse(ColumnOriginX[IndexOffset + z * NParticles + i]) - ShiftPriors[i].X,
float.Parse(ColumnOriginY[IndexOffset + z * NParticles + i]) - ShiftPriors[i].Y);
}
Image AverageFT = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true, true);
Image AveragePS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
Image Weights = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
Weights.Fill(1e-6f);
Image FrameParticles = new Image(IntPtr.Zero, new int3(DimsRegion.X * 2, DimsRegion.Y * 2, NParticles));
float StepZ = 1f / Math.Max(Dims.Z - 1, 1);
for (int z = 0; z < Dims.Z; z++)
{
float CoordZ = z * StepZ;
/*GPU.Extract(OriginalStack.GetDeviceSlice(z, Intent.Read),
FrameParticles.GetDevice(Intent.Write),
Dims.Slice(),
new int3(DimsRegion.X * 2, DimsRegion.Y * 2, 1),
Helper.ToInterleaved(Origins),
(uint)NParticles);*/
// Shift particles
{
float3[] Shifts = new float3[NParticles];
for (int i = 0; i < NParticles; i++)
{
float NormX = Math.Max(0.15f, Math.Min((float)Origins[i].X / Dims.X, 0.85f));
float NormY = Math.Max(0.15f, Math.Min((float)Origins[i].Y / Dims.Y, 0.85f));
float3 Coords = new float3(NormX, NormY, CoordZ);
Shifts[i] = new float3(GridMovementX.GetInterpolated(Coords) + ParticleTracks[i][z].X,
GridMovementY.GetInterpolated(Coords) + ParticleTracks[i][z].Y,
0f);
}
FrameParticles.ShiftSlices(Shifts);
}
Image FrameParticlesCropped = FrameParticles.AsPadded(new int2(DimsRegion));
Image FrameParticlesFT = FrameParticlesCropped.AsFFT();
FrameParticlesCropped.Dispose();
Image PS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
PS.Fill(1f);
// Apply motion blur filter.
{
const int Samples = 11;
float StartZ = (z - 0.75f) * StepZ;
float StopZ = (z + 0.75f) * StepZ;
float2[] Shifts = new float2[Samples * NParticles];
for (int p = 0; p < NParticles; p++)
{
float NormX = Math.Max(0.15f, Math.Min((float)Origins[p].X / Dims.X, 0.85f));
float NormY = Math.Max(0.15f, Math.Min((float)Origins[p].Y / Dims.Y, 0.85f));
for (int zz = 0; zz < Samples; zz++)
{
float zp = StartZ + (StopZ - StartZ) / (Samples - 1) * zz;
float3 Coords = new float3(NormX, NormY, zp);
Shifts[p * Samples + zz] = new float2(GridMovementX.GetInterpolated(Coords),
GridMovementY.GetInterpolated(Coords));
}
}
Image MotionFilter = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
GPU.CreateMotionBlur(MotionFilter.GetDevice(Intent.Write),
DimsRegion,
Helper.ToInterleaved(Shifts.Select(v => new float3(v.X, v.Y, 0)).ToArray()),
Samples,
(uint)NParticles);
PS.Multiply(MotionFilter);
//MotionFilter.WriteMRC("motion.mrc");
MotionFilter.Dispose();
}
// Apply CTF.
if (CTF != null)
{
CTFStruct[] Structs = new CTFStruct[NParticles];
for (int p = 0; p < NParticles; p++)
{
CTF Altered = CTF.GetCopy();
Altered.Defocus = (decimal)GridCTF.GetInterpolated(new float3(float.Parse(ColumnPosX[RowIndices[p]]) / Dims.X,
float.Parse(ColumnPosY[RowIndices[p]]) / Dims.Y,
z * StepZ));
Structs[p] = Altered.ToStruct();
}
Image CTFImage = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
GPU.CreateCTF(CTFImage.GetDevice(Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CTFCoords.ElementsSliceComplex,
Structs,
false,
(uint)NParticles);
//CTFImage.Abs();
PS.Multiply(CTFImage);
//CTFImage.WriteMRC("ctf.mrc");
CTFImage.Dispose();
}
// Apply dose weighting.
/*{
float3 NikoConst = new float3(0.245f, -1.665f, 2.81f);
// Niko's formula expects e-/A2/frame, we've got e-/px/frame -- convert!
float FrameDose = (float)MainWindow.Options.CorrectDosePerFrame * (z + 0.5f) / (PixelSize * PixelSize);
Image DoseImage = new Image(IntPtr.Zero, DimsRegion, true);
GPU.DoseWeighting(CTFFreq.GetDevice(Intent.Read),
DoseImage.GetDevice(Intent.Write),
(uint)DoseImage.ElementsSliceComplex,
new[] { FrameDose },
NikoConst,
1);
PS.MultiplySlices(DoseImage);
//DoseImage.WriteMRC("dose.mrc");
DoseImage.Dispose();
}*/
// Copy custom CTF into the CTF stack
GPU.CopyDeviceToDevice(PS.GetDevice(Intent.Read),
new IntPtr((long)CTFStack.GetDevice(Intent.Write) + CTFStack.ElementsSliceReal * NParticles * z * sizeof (float)),
CTFStack.ElementsSliceReal * NParticles);
Image PSAbs = new Image(PS.GetDevice(Intent.Read), new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
PSAbs.Abs();
FrameParticlesFT.Multiply(PSAbs);
AverageFT.Add(FrameParticlesFT);
Weights.Add(PSAbs);
PS.Multiply(PSAbs);
AveragePS.Add(PS);
PS.Dispose();
FrameParticlesFT.Dispose();
PSAbs.Dispose();
// Write paths to custom CTFs into the .star
for (int i = 0; i < NParticles; i++)
{
string ParticleCTFPath = (CTFStackIndex + 1).ToString("D6") + "@particlectf/" + RootName + ".mrcs";
table.SetRowValue(RowIndices[NParticles * z + i], "rlnCtfImage", ParticleCTFPath);
CTFStackIndex++;
}
}
FrameParticles.Dispose();
CTFCoords.Dispose();
CTFFreq.Dispose();
AverageFT.Divide(Weights);
AveragePS.Divide(Weights);
Weights.Dispose();
Image AverageParticles = AverageFT.AsIFFT();
AverageFT.Dispose();
GPU.NormParticles(AverageParticles.GetDevice(Intent.Read), AverageParticles.GetDevice(Intent.Write), DimsRegion, (uint)(90f / PixelSize), true, (uint)NParticles);
HeaderMRC ParticlesHeader = new HeaderMRC
{
Pixelsize = new float3(PixelSize, PixelSize, PixelSize)
};
AverageParticles.WriteMRC(ParticlesPath, ParticlesHeader);
AverageParticles.Dispose();
//OriginalStack.Dispose();
CTFStack.WriteMRC(DirectoryName + "particlectf/" + RootName + ".mrcs");
CTFStack.Dispose();
/*for (int i = 0; i < NParticles; i++)
{
string ParticlePath = (i + 1).ToString("D6") + "@particles/" + RootName + "_particles.mrcs";
table.SetRowValue(RowIndices[i], "rlnImageName", ParticlePath);
}*/
AveragePS.Dispose();
//table.RemoveRows(RowIndices.Skip(NParticles).ToArray());
}
public void ExportParticlesMovie(Star tableIn, Star tableOut, MapHeader originalHeader, Image originalStack, int size, float particleradius, decimal scaleFactor)
{
int CurrentDevice = GPU.GetDevice();
#region Make sure directories exist.
lock (tableIn)
{
if (!Directory.Exists(ParticleMoviesDir))
Directory.CreateDirectory(ParticleMoviesDir);
if (!Directory.Exists(ParticleCTFMoviesDir))
Directory.CreateDirectory(ParticleCTFMoviesDir);
}
#endregion
#region Get row indices for all, and individual halves
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName))
RowIndices.Add(i);
//RowIndices = RowIndices.Take(13).ToList();
List<int> RowIndices1 = new List<int>();
List<int> RowIndices2 = new List<int>();
for (int i = 0; i < RowIndices.Count; i++)
if (tableIn.GetRowValue(RowIndices[i], "rlnRandomSubset") == "1")
RowIndices1.Add(RowIndices[i]);
else
RowIndices2.Add(RowIndices[i]);
#endregion
if (RowIndices.Count == 0)
return;
#region Auxiliary variables
List<int> TableOutIndices = new List<int>();
int3 Dims = originalHeader.Dimensions;
Dims.Z = 36;
int3 DimsRegion = new int3(size, size, 1);
int3 DimsPadded = new int3(size * 2, size * 2, 1);
int NParticles = RowIndices.Count;
int NParticles1 = RowIndices1.Count;
int NParticles2 = RowIndices2.Count;
float PixelSize = (float)CTF.PixelSize / 1.00f;
float PixelDelta = (float)CTF.PixelSizeDelta / 1.00f;
float PixelAngle = (float)CTF.PixelSizeAngle * Helper.ToRad;
#endregion
#region Prepare initial coordinates and shifts
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
int3[] Origins1 = new int3[NParticles1];
int3[] Origins2 = new int3[NParticles2];
float3[] ResidualShifts1 = new float3[NParticles1];
float3[] ResidualShifts2 = new float3[NParticles2];
lock (tableIn) // Writing to the table, better be on the safe side
{
// Half1: Add translational shifts to coordinates, sans the fractional part
for (int i = 0; i < NParticles1; i++)
{
float2 Pos = new float2(float.Parse(ColumnPosX[RowIndices1[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices1[i]], CultureInfo.InvariantCulture)) * 1.00f;
float2 Shift = new float2(float.Parse(ColumnOriginX[RowIndices1[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices1[i]], CultureInfo.InvariantCulture)) * 1.00f;
Origins1[i] = new int3((int)(Pos.X - Shift.X),
(int)(Pos.Y - Shift.Y),
0);
ResidualShifts1[i] = new float3(-MathHelper.ResidualFraction(Pos.X - Shift.X),
-MathHelper.ResidualFraction(Pos.Y - Shift.Y),
0f);
tableIn.SetRowValue(RowIndices1[i], "rlnCoordinateX", Origins1[i].X.ToString());
tableIn.SetRowValue(RowIndices1[i], "rlnCoordinateY", Origins1[i].Y.ToString());
tableIn.SetRowValue(RowIndices1[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices1[i], "rlnOriginY", "0.0");
}
// Half2: Add translational shifts to coordinates, sans the fractional part
for (int i = 0; i < NParticles2; i++)
{
float2 Pos = new float2(float.Parse(ColumnPosX[RowIndices2[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices2[i]], CultureInfo.InvariantCulture)) * 1.00f;
float2 Shift = new float2(float.Parse(ColumnOriginX[RowIndices2[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices2[i]], CultureInfo.InvariantCulture)) * 1.00f;
Origins2[i] = new int3((int)(Pos.X - Shift.X),
(int)(Pos.Y - Shift.Y),
0);
ResidualShifts2[i] = new float3(-MathHelper.ResidualFraction(Pos.X - Shift.X),
-MathHelper.ResidualFraction(Pos.Y - Shift.Y),
0f);
tableIn.SetRowValue(RowIndices2[i], "rlnCoordinateX", Origins2[i].X.ToString());
tableIn.SetRowValue(RowIndices2[i], "rlnCoordinateY", Origins2[i].Y.ToString());
tableIn.SetRowValue(RowIndices2[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices2[i], "rlnOriginY", "0.0");
}
}
#endregion
#region Allocate memory for particle and PS stacks
Image ParticleStackAll = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles * Dims.Z));
Image ParticleStack1 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles1 * Dims.Z));
Image ParticleStack2 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles2 * Dims.Z));
Image PSStackAll = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles * Dims.Z), true);
Image PSStack1 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles1 * Dims.Z), true);
Image PSStack2 = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles2 * Dims.Z), true);
Image FrameParticles1 = new Image(IntPtr.Zero, new int3(DimsPadded.X, DimsPadded.Y, NParticles1));
Image FrameParticles2 = new Image(IntPtr.Zero, new int3(DimsPadded.X, DimsPadded.Y, NParticles2));
float[][] ParticleStackData = ParticleStackAll.GetHost(Intent.Write);
float[][] ParticleStackData1 = ParticleStack1.GetHost(Intent.Write);
float[][] ParticleStackData2 = ParticleStack2.GetHost(Intent.Write);
float[][] PSStackData = PSStackAll.GetHost(Intent.Write);
float[][] PSStackData1 = PSStack1.GetHost(Intent.Write);
float[][] PSStackData2 = PSStack2.GetHost(Intent.Write);
#endregion
#region Create rows in outTable
lock (tableOut) // Creating rows in outTable, this absolutely needs to be staged sequentially
{
for (int z = 0; z < Dims.Z; z++)
{
for (int i = 0; i < NParticles; i++)
{
int Index = i < NParticles1 ? RowIndices1[i] : RowIndices2[i - NParticles1];
string OriParticlePath = (i + 1).ToString("D6") + "@particles/" + RootName + "_particles.mrcs";
string ParticleName = (z * NParticles + i + 1).ToString("D6") + "@particlemovies/" + RootName + "_particles.mrcs";
string ParticleCTFName = (z * NParticles + i + 1).ToString("D6") + "@particlectfmovies/" + RootName + "_particlectf.mrcs";
List<string> NewRow = tableIn.GetRow(Index).Select(v => v).ToList(); // Get copy of original row.
NewRow[tableOut.GetColumnIndex("rlnOriginalParticleName")] = OriParticlePath;
NewRow[tableOut.GetColumnIndex("rlnAngleRotPrior")] = tableIn.GetRowValue(Index, "rlnAngleRot");
NewRow[tableOut.GetColumnIndex("rlnAngleTiltPrior")] = tableIn.GetRowValue(Index, "rlnAngleTilt");
NewRow[tableOut.GetColumnIndex("rlnAnglePsiPrior")] = tableIn.GetRowValue(Index, "rlnAnglePsi");
NewRow[tableOut.GetColumnIndex("rlnOriginXPrior")] = "0.0";
NewRow[tableOut.GetColumnIndex("rlnOriginYPrior")] = "0.0";
NewRow[tableOut.GetColumnIndex("rlnImageName")] = ParticleName;
NewRow[tableOut.GetColumnIndex("rlnCtfImage")] = ParticleCTFName;
NewRow[tableOut.GetColumnIndex("rlnMicrographName")] = (z + 1).ToString("D6") + "@stack/" + RootName + "_movie.mrcs";
TableOutIndices.Add(tableOut.RowCount);
tableOut.AddRow(NewRow);
}
}
}
#endregion
#region For every frame, extract particles from each half; shift, correct, and norm them
float StepZ = 1f / Math.Max(Dims.Z - 1, 1);
for (int z = 0; z < Dims.Z; z++)
{
float CoordZ = z * StepZ;
#region Extract, correct, and norm particles
#region Half 1
{
if (originalStack != null)
GPU.Extract(originalStack.GetDeviceSlice(z, Intent.Read),
FrameParticles1.GetDevice(Intent.Write),
Dims.Slice(),
DimsPadded,
Helper.ToInterleaved(Origins1.Select(v => new int3(v.X - DimsPadded.X / 2, v.Y - DimsPadded.Y / 2, 0)).ToArray()),
(uint)NParticles1);
// Shift particles
{
float3[] Shifts = new float3[NParticles1];
for (int i = 0; i < NParticles1; i++)
{
float3 Coords = new float3((float)Origins1[i].X / Dims.X, (float)Origins1[i].Y / Dims.Y, CoordZ);
Shifts[i] = ResidualShifts1[i] + new float3(GetShiftFromPyramid(Coords)) * 1.00f;
}
FrameParticles1.ShiftSlices(Shifts);
}
Image FrameParticlesCropped = FrameParticles1.AsPadded(new int2(DimsRegion));
Image FrameParticlesCorrected = FrameParticlesCropped.AsAnisotropyCorrected(new int2(DimsRegion),
PixelSize + PixelDelta / 2f,
PixelSize - PixelDelta / 2f,
PixelAngle,
6);
FrameParticlesCropped.Dispose();
GPU.NormParticles(FrameParticlesCorrected.GetDevice(Intent.Read),
FrameParticlesCorrected.GetDevice(Intent.Write),
DimsRegion,
(uint)(particleradius / PixelSize),
true,
(uint)NParticles1);
float[][] FrameParticlesCorrectedData = FrameParticlesCorrected.GetHost(Intent.Read);
for (int n = 0; n < NParticles1; n++)
{
ParticleStackData[z * NParticles + n] = FrameParticlesCorrectedData[n];
ParticleStackData1[z * NParticles1 + n] = FrameParticlesCorrectedData[n];
}
//FrameParticlesCorrected.WriteMRC("intermediate_particles1.mrc");
FrameParticlesCorrected.Dispose();
}
#endregion
#region Half 2
{
if (originalStack != null)
GPU.Extract(originalStack.GetDeviceSlice(z, Intent.Read),
FrameParticles2.GetDevice(Intent.Write),
Dims.Slice(),
DimsPadded,
Helper.ToInterleaved(Origins2.Select(v => new int3(v.X - DimsPadded.X / 2, v.Y - DimsPadded.Y / 2, 0)).ToArray()),
(uint)NParticles2);
// Shift particles
{
float3[] Shifts = new float3[NParticles2];
for (int i = 0; i < NParticles2; i++)
{
float3 Coords = new float3((float)Origins2[i].X / Dims.X, (float)Origins2[i].Y / Dims.Y, CoordZ);
Shifts[i] = ResidualShifts2[i] + new float3(GetShiftFromPyramid(Coords)) * 1.00f;
}
FrameParticles2.ShiftSlices(Shifts);
}
Image FrameParticlesCropped = FrameParticles2.AsPadded(new int2(DimsRegion));
Image FrameParticlesCorrected = FrameParticlesCropped.AsAnisotropyCorrected(new int2(DimsRegion),
PixelSize + PixelDelta / 2f,
PixelSize - PixelDelta / 2f,
PixelAngle,
6);
FrameParticlesCropped.Dispose();
GPU.NormParticles(FrameParticlesCorrected.GetDevice(Intent.Read),
FrameParticlesCorrected.GetDevice(Intent.Write),
DimsRegion,
(uint)(particleradius / PixelSize),
true,
(uint)NParticles2);
float[][] FrameParticlesCorrectedData = FrameParticlesCorrected.GetHost(Intent.Read);
for (int n = 0; n < NParticles2; n++)
{
ParticleStackData[z * NParticles + NParticles1 + n] = FrameParticlesCorrectedData[n];
ParticleStackData2[z * NParticles2 + n] = FrameParticlesCorrectedData[n];
}
//FrameParticlesCorrected.WriteMRC("intermediate_particles2.mrc");
FrameParticlesCorrected.Dispose();
}
#endregion
#endregion
#region PS Half 1
{
Image PS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles1), true);
PS.Fill(1f);
// Apply motion blur filter.
#region Motion blur weighting
{
const int Samples = 11;
float StartZ = (z - 0.5f) * StepZ;
float StopZ = (z + 0.5f) * StepZ;
float2[] Shifts = new float2[Samples * NParticles1];
for (int p = 0; p < NParticles1; p++)
{
float NormX = (float)Origins1[p].X / Dims.X;
float NormY = (float)Origins1[p].Y / Dims.Y;
for (int zz = 0; zz < Samples; zz++)
{
float zp = StartZ + (StopZ - StartZ) / (Samples - 1) * zz;
float3 Coords = new float3(NormX, NormY, zp);
Shifts[p * Samples + zz] = GetShiftFromPyramid(Coords) * 1.00f;
}
}
Image MotionFilter = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles1), true);
GPU.CreateMotionBlur(MotionFilter.GetDevice(Intent.Write),
DimsRegion,
Helper.ToInterleaved(Shifts.Select(v => new float3(v.X, v.Y, 0)).ToArray()),
Samples,
(uint)NParticles1);
PS.Multiply(MotionFilter);
//MotionFilter.WriteMRC("motion.mrc");
MotionFilter.Dispose();
}
#endregion
float[][] PSData = PS.GetHost(Intent.Read);
for (int n = 0; n < NParticles1; n++)
PSStackData[z * NParticles + n] = PSData[n];
//PS.WriteMRC("intermediate_ps1.mrc");
PS.Dispose();
}
#endregion
#region PS Half 2
{
Image PS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles2), true);
PS.Fill(1f);
// Apply motion blur filter.
#region Motion blur weighting
{
const int Samples = 11;
float StartZ = (z - 0.5f) * StepZ;
float StopZ = (z + 0.5f) * StepZ;
float2[] Shifts = new float2[Samples * NParticles2];
for (int p = 0; p < NParticles2; p++)
{
float NormX = (float)Origins2[p].X / Dims.X;
float NormY = (float)Origins2[p].Y / Dims.Y;
for (int zz = 0; zz < Samples; zz++)
{
float zp = StartZ + (StopZ - StartZ) / (Samples - 1) * zz;
float3 Coords = new float3(NormX, NormY, zp);
Shifts[p * Samples + zz] = GetShiftFromPyramid(Coords) * 1.00f;
}
}
Image MotionFilter = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles2), true);
GPU.CreateMotionBlur(MotionFilter.GetDevice(Intent.Write),
DimsRegion,
Helper.ToInterleaved(Shifts.Select(v => new float3(v.X, v.Y, 0)).ToArray()),
Samples,
(uint)NParticles2);
PS.Multiply(MotionFilter);
//MotionFilter.WriteMRC("motion.mrc");
MotionFilter.Dispose();
}
#endregion
float[][] PSData = PS.GetHost(Intent.Read);
for (int n = 0; n < NParticles2; n++)
PSStackData[z * NParticles + NParticles1 + n] = PSData[n];
//PS.WriteMRC("intermediate_ps2.mrc");
PS.Dispose();
}
#endregion
}
FrameParticles1.Dispose();
FrameParticles2.Dispose();
originalStack.FreeDevice();
#endregion
HeaderMRC ParticlesHeader = new HeaderMRC
{
Pixelsize = new float3(PixelSize, PixelSize, PixelSize)
};
// Do translation and rotation BFGS per particle
{
float MaxHigh = 2.6f;
CubicGrid GridX = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridY = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridRot = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridTilt = new CubicGrid(new int3(NParticles1, 1, 2));
CubicGrid GridPsi = new CubicGrid(new int3(NParticles1, 1, 2));
int2 DimsCropped = new int2(DimsRegion / (MaxHigh / PixelSize / 2f)) / 2 * 2;
#region Get coordinates for CTF and Fourier-space shifts
Image CTFCoords;
Image ShiftFactors;
{
float2[] CTFCoordsData = new float2[(DimsCropped.X / 2 + 1) * DimsCropped.Y];
float2[] ShiftFactorsData = new float2[(DimsCropped.X / 2 + 1) * DimsCropped.Y];
for (int y = 0; y < DimsCropped.Y; y++)
for (int x = 0; x < DimsCropped.X / 2 + 1; x++)
{
int xx = x;
int yy = y < DimsCropped.Y / 2 + 1 ? y : y - DimsCropped.Y;
float xs = xx / (float)DimsRegion.X;
float ys = yy / (float)DimsRegion.Y;
float r = (float)Math.Sqrt(xs * xs + ys * ys);
float angle = (float)(Math.Atan2(yy, xx));
CTFCoordsData[y * (DimsCropped.X / 2 + 1) + x] = new float2(r / PixelSize, angle);
ShiftFactorsData[y * (DimsCropped.X / 2 + 1) + x] = new float2((float)-xx / DimsRegion.X * 2f * (float)Math.PI,
(float)-yy / DimsRegion.X * 2f * (float)Math.PI);
}
CTFCoords = new Image(CTFCoordsData, new int3(DimsCropped), true);
ShiftFactors = new Image(ShiftFactorsData, new int3(DimsCropped), true);
}
#endregion
#region Get inverse sigma2 spectrum for this micrograph from Relion's model.star
Image Sigma2Noise = new Image(new int3(DimsCropped), true);
{
int GroupNumber = int.Parse(tableIn.GetRowValue(RowIndices[0], "rlnGroupNumber"));
//Star SigmaTable = new Star("D:\\rado27\\Refine3D\\run1_ct5_it009_half1_model.star", "data_model_group_" + GroupNumber);
Star SigmaTable = new Star(MainWindow.Options.ModelStarPath, "data_model_group_" + GroupNumber);
float[] SigmaValues = SigmaTable.GetColumn("rlnSigma2Noise").Select(v => float.Parse(v)).ToArray();
float[] Sigma2NoiseData = Sigma2Noise.GetHost(Intent.Write)[0];
Helper.ForEachElementFT(DimsCropped, (x, y, xx, yy, r, angle) =>
{
int ir = (int)r;
float val = 0;
if (ir < SigmaValues.Length && ir >= size / (50f / PixelSize) && ir < DimsCropped.X / 2)
{
if (SigmaValues[ir] != 0f)
val = 1f / SigmaValues[ir];
}
Sigma2NoiseData[y * (DimsCropped.X / 2 + 1) + x] = val;
});
float MaxSigma = MathHelper.Max(Sigma2NoiseData);
for (int i = 0; i < Sigma2NoiseData.Length; i++)
Sigma2NoiseData[i] /= MaxSigma;
Sigma2Noise.RemapToFT();
}
//Sigma2Noise.WriteMRC("d_sigma2noise.mrc");
#endregion
#region Initialize particle angles for both halves
float3[] ParticleAngles1 = new float3[NParticles1];
float3[] ParticleAngles2 = new float3[NParticles2];
for (int p = 0; p < NParticles1; p++)
ParticleAngles1[p] = new float3(float.Parse(tableIn.GetRowValue(RowIndices1[p], "rlnAngleRot")),
float.Parse(tableIn.GetRowValue(RowIndices1[p], "rlnAngleTilt")),
float.Parse(tableIn.GetRowValue(RowIndices1[p], "rlnAnglePsi")));
for (int p = 0; p < NParticles2; p++)
ParticleAngles2[p] = new float3(float.Parse(tableIn.GetRowValue(RowIndices2[p], "rlnAngleRot")),
float.Parse(tableIn.GetRowValue(RowIndices2[p], "rlnAngleTilt")),
float.Parse(tableIn.GetRowValue(RowIndices2[p], "rlnAnglePsi")));
#endregion
#region Prepare masks
Image Masks1, Masks2;
{
// Half 1
{
Image Volume = StageDataLoad.LoadMap(MainWindow.Options.MaskPath, new int2(1, 1), 0, typeof (float));
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
Image VolMaskFT = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
Image MasksFT = VolMaskFT.AsProjections(ParticleAngles1.Select(v => new float3(v.X * Helper.ToRad, v.Y * Helper.ToRad, v.Z * Helper.ToRad)).ToArray(),
new int2(DimsRegion),
MainWindow.Options.ProjectionOversample);
VolMaskFT.Dispose();
Masks1 = MasksFT.AsIFFT();
MasksFT.Dispose();
Masks1.RemapFromFT();
Parallel.ForEach(Masks1.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(50f, slice[i])) - 2) / 48f;
});
}
// Half 2
{
Image Volume = StageDataLoad.LoadMap(MainWindow.Options.MaskPath, new int2(1, 1), 0, typeof(float));
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
Image VolMaskFT = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
Image MasksFT = VolMaskFT.AsProjections(ParticleAngles2.Select(v => new float3(v.X * Helper.ToRad, v.Y * Helper.ToRad, v.Z * Helper.ToRad)).ToArray(),
new int2(DimsRegion),
MainWindow.Options.ProjectionOversample);
VolMaskFT.Dispose();
Masks2 = MasksFT.AsIFFT();
MasksFT.Dispose();
Masks2.RemapFromFT();
Parallel.ForEach(Masks2.GetHost(Intent.ReadWrite), slice =>
{
for (int i = 0; i < slice.Length; i++)
slice[i] = (Math.Max(2f, Math.Min(50f, slice[i])) - 2) / 48f;
});
}
}
//Masks1.WriteMRC("d_masks1.mrc");
//Masks2.WriteMRC("d_masks2.mrc");
#endregion
#region Load and prepare references for both halves
Image VolRefFT1;
{
Image Volume = StageDataLoad.LoadMap(MainWindow.Options.ReferencePath, new int2(1, 1), 0, typeof(float));
//GPU.Normalize(Volume.GetDevice(Intent.Read), Volume.GetDevice(Intent.Write), (uint)Volume.ElementsReal, 1);
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
VolRefFT1 = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
}
VolRefFT1.FreeDevice();
Image VolRefFT2;
{
// Can't assume there is a second half, but certainly hope so
string Half2Path = MainWindow.Options.ReferencePath;
if (Half2Path.Contains("half1"))
Half2Path = Half2Path.Replace("half1", "half2");
Image Volume = StageDataLoad.LoadMap(Half2Path, new int2(1, 1), 0, typeof(float));
//GPU.Normalize(Volume.GetDevice(Intent.Read), Volume.GetDevice(Intent.Write), (uint)Volume.ElementsReal, 1);
Image VolumePadded = Volume.AsPadded(Volume.Dims * MainWindow.Options.ProjectionOversample);
Volume.Dispose();
VolumePadded.RemapToFT(true);
VolRefFT2 = VolumePadded.AsFFT(true);
VolumePadded.Dispose();
}
VolRefFT2.FreeDevice();
#endregion
#region Prepare particles: group and resize to DimsCropped
Image ParticleStackFT1 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles1 * Dims.Z / 3), true, true);
{
GPU.CreatePolishing(ParticleStack1.GetDevice(Intent.Read),
ParticleStackFT1.GetDevice(Intent.Write),
Masks1.GetDevice(Intent.Read),
new int2(DimsRegion),
DimsCropped,
NParticles1,
Dims.Z);
ParticleStack1.FreeDevice();
Masks1.Dispose();
/*Image Amps = ParticleStackFT1.AsIFFT();
Amps.RemapFromFT();
Amps.WriteMRC("d_particlestackft1.mrc");
Amps.Dispose();*/
}
Image ParticleStackFT2 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles2 * Dims.Z / 3), true, true);
{
GPU.CreatePolishing(ParticleStack2.GetDevice(Intent.Read),
ParticleStackFT2.GetDevice(Intent.Write),
Masks2.GetDevice(Intent.Read),
new int2(DimsRegion),
DimsCropped,
NParticles2,
Dims.Z);
ParticleStack1.FreeDevice();
Masks2.Dispose();
/*Image Amps = ParticleStackFT2.AsIFFT();
Amps.RemapFromFT();
Amps.WriteMRC("d_particlestackft2.mrc");
Amps.Dispose();*/
}
#endregion
Image Projections1 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles1 * Dims.Z / 3), true, true);
Image Projections2 = new Image(IntPtr.Zero, new int3(DimsCropped.X, DimsCropped.Y, NParticles2 * Dims.Z / 3), true, true);
Image Shifts1 = new Image(new int3(NParticles1, Dims.Z / 3, 1), false, true);
float3[] Angles1 = new float3[NParticles1 * Dims.Z / 3];
CTFStruct[] CTFParams1 = new CTFStruct[NParticles1 * Dims.Z / 3];
Image Shifts2 = new Image(new int3(NParticles2, Dims.Z / 3, 1), false, true);
float3[] Angles2 = new float3[NParticles2 * Dims.Z / 3];
CTFStruct[] CTFParams2 = new CTFStruct[NParticles2 * Dims.Z / 3];
float[] BFacs =
{
-3.86f,
0.00f,
-17.60f,
-35.24f,
-57.48f,
-93.51f,
-139.57f,
-139.16f
};
#region Initialize defocus and phase shift values
float[] InitialDefoci1 = new float[NParticles1 * (Dims.Z / 3)];
float[] InitialPhaseShifts1 = new float[NParticles1 * (Dims.Z / 3)];
float[] InitialDefoci2 = new float[NParticles2 * (Dims.Z / 3)];
float[] InitialPhaseShifts2 = new float[NParticles2 * (Dims.Z / 3)];
for (int z = 0, i = 0; z < Dims.Z / 3; z++)
{
for (int p = 0; p < NParticles1; p++, i++)
{
InitialDefoci1[i] = GridCTF.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
InitialPhaseShifts1[i] = GridCTFPhase.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
CTF Alt = CTF.GetCopy();
Alt.PixelSize = (decimal)PixelSize;
Alt.PixelSizeDelta = 0;
Alt.Defocus = (decimal)InitialDefoci1[i];
Alt.PhaseShift = (decimal)InitialPhaseShifts1[i];
//Alt.Bfactor = (decimal)BFacs[z];
CTFParams1[i] = Alt.ToStruct();
}
}
for (int z = 0, i = 0; z < Dims.Z / 3; z++)
{
for (int p = 0; p < NParticles2; p++, i++)
{
InitialDefoci2[i] = GridCTF.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
InitialPhaseShifts2[i] = GridCTFPhase.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)(z * 3 + 1) / (Dims.Z - 1)));
CTF Alt = CTF.GetCopy();
Alt.PixelSize = (decimal)PixelSize;
Alt.PixelSizeDelta = 0;
Alt.Defocus = (decimal)InitialDefoci2[i];
Alt.PhaseShift = (decimal)InitialPhaseShifts2[i];
//Alt.Bfactor = (decimal)BFacs[z];
CTFParams2[i] = Alt.ToStruct();
}
}
#endregion
#region SetPositions lambda
Action<double[]> SetPositions = input =>
{
float BorderZ = 0.5f / (Dims.Z / 3);
GridX = new CubicGrid(new int3(NParticles, 1, 2), input.Take(NParticles * 2).Select(v => (float)v).ToArray());
GridY = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 1).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredX = GridX.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] AlteredY = GridY.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
GridRot = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 2).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridTilt = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 3).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridPsi = new CubicGrid(new int3(NParticles, 1, 2), input.Skip(NParticles * 2 * 4).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredRot = GridRot.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] AlteredTilt = GridTilt.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] AlteredPsi = GridPsi.GetInterpolatedNative(new int3(NParticles, 1, Dims.Z / 3), new float3(0, 0, BorderZ));
float[] ShiftData1 = Shifts1.GetHost(Intent.Write)[0];
float[] ShiftData2 = Shifts2.GetHost(Intent.Write)[0];
for (int z = 0; z < Dims.Z / 3; z++)
{
// Half 1
for (int p = 0; p < NParticles1; p++)
{
int i1 = z * NParticles1 + p;
int i = z * NParticles + p;
ShiftData1[i1 * 2] = AlteredX[i];
ShiftData1[i1 * 2 + 1] = AlteredY[i];
Angles1[i1] = new float3(AlteredRot[i] * 1f * Helper.ToRad, AlteredTilt[i] * 1f * Helper.ToRad, AlteredPsi[i] * 1f * Helper.ToRad);
}
// Half 2
for (int p = 0; p < NParticles2; p++)
{
int i2 = z * NParticles2 + p;
int i = z * NParticles + NParticles1 + p;
ShiftData2[i2 * 2] = AlteredX[i];
ShiftData2[i2 * 2 + 1] = AlteredY[i];
Angles2[i2] = new float3(AlteredRot[i] * 1f * Helper.ToRad, AlteredTilt[i] * 1f * Helper.ToRad, AlteredPsi[i] * 1f * Helper.ToRad);
}
}
};
#endregion
#region EvalIndividuals lambda
Func<double[], bool, double[]> EvalIndividuals = (input, redoProj) =>
{
SetPositions(input);
if (redoProj)
{
GPU.ProjectForward(VolRefFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Write),
VolRefFT1.Dims,
DimsCropped,
Helper.ToInterleaved(Angles1),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles1 * Dims.Z / 3));
GPU.ProjectForward(VolRefFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Write),
VolRefFT2.Dims,
DimsCropped,
Helper.ToInterleaved(Angles2),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles2 * Dims.Z / 3));
}
/*{
Image ProjectionsAmps = Projections1.AsIFFT();
ProjectionsAmps.RemapFromFT();
ProjectionsAmps.WriteMRC("d_projectionsamps1.mrc");
ProjectionsAmps.Dispose();
}
{
Image ProjectionsAmps = Projections2.AsIFFT();
ProjectionsAmps.RemapFromFT();
ProjectionsAmps.WriteMRC("d_projectionsamps2.mrc");
ProjectionsAmps.Dispose();
}*/
float[] Diff1 = new float[NParticles1];
float[] DiffAll1 = new float[NParticles1 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams1,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts1.GetDevice(Intent.Read),
Diff1,
DiffAll1,
(uint)NParticles1,
(uint)Dims.Z / 3);
float[] Diff2 = new float[NParticles2];
float[] DiffAll2 = new float[NParticles2 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams2,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts2.GetDevice(Intent.Read),
Diff2,
DiffAll2,
(uint)NParticles2,
(uint)Dims.Z / 3);
double[] DiffBoth = new double[NParticles];
for (int p = 0; p < NParticles1; p++)
DiffBoth[p] = Diff1[p];
for (int p = 0; p < NParticles2; p++)
DiffBoth[NParticles1 + p] = Diff2[p];
return DiffBoth;
};
#endregion
Func<double[], double> Eval = input =>
{
float Result = MathHelper.Mean(EvalIndividuals(input, true).Select(v => (float)v)) * NParticles;
Debug.WriteLine(Result);
return Result;
};
Func<double[], double[]> Grad = input =>
{
SetPositions(input);
GPU.ProjectForward(VolRefFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Write),
VolRefFT1.Dims,
DimsCropped,
Helper.ToInterleaved(Angles1),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles1 * Dims.Z / 3));
GPU.ProjectForward(VolRefFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Write),
VolRefFT2.Dims,
DimsCropped,
Helper.ToInterleaved(Angles2),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles2 * Dims.Z / 3));
double[] Result = new double[input.Length];
double Step = 0.1;
int NVariables = 10; // (Shift + Euler) * 2
for (int v = 0; v < NVariables; v++)
{
double[] InputPlus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
int iv = i / NParticles;
if (iv == v)
InputPlus[i] = input[i] + Step;
else
InputPlus[i] = input[i];
}
double[] ScorePlus = EvalIndividuals(InputPlus, v >= 4);
double[] InputMinus = new double[input.Length];
for (int i = 0; i < input.Length; i++)
{
int iv = i / NParticles;
if (iv == v)
InputMinus[i] = input[i] - Step;
else
InputMinus[i] = input[i];
}
double[] ScoreMinus = EvalIndividuals(InputMinus, v >= 4);
for (int i = 0; i < NParticles; i++)
Result[v * NParticles + i] = (ScorePlus[i] - ScoreMinus[i]) / (Step * 2.0);
}
return Result;
};
double[] StartParams = new double[NParticles * 2 * 5];
for (int i = 0; i < NParticles * 2; i++)
{
int p = i % NParticles;
StartParams[NParticles * 2 * 0 + i] = 0;
StartParams[NParticles * 2 * 1 + i] = 0;
if (p < NParticles1)
{
StartParams[NParticles * 2 * 2 + i] = ParticleAngles1[p].X / 1.0;
StartParams[NParticles * 2 * 3 + i] = ParticleAngles1[p].Y / 1.0;
StartParams[NParticles * 2 * 4 + i] = ParticleAngles1[p].Z / 1.0;
}
else
{
p -= NParticles1;
StartParams[NParticles * 2 * 2 + i] = ParticleAngles2[p].X / 1.0;
StartParams[NParticles * 2 * 3 + i] = ParticleAngles2[p].Y / 1.0;
StartParams[NParticles * 2 * 4 + i] = ParticleAngles2[p].Z / 1.0;
}
}
BroydenFletcherGoldfarbShanno Optimizer = new BroydenFletcherGoldfarbShanno(StartParams.Length, Eval, Grad);
Optimizer.Epsilon = 3e-7;
Optimizer.Maximize(StartParams);
#region Calculate particle quality for high frequencies
float[] ParticleQuality = new float[NParticles * (Dims.Z / 3)];
{
Sigma2Noise.Dispose();
Sigma2Noise = new Image(new int3(DimsCropped), true);
{
int GroupNumber = int.Parse(tableIn.GetRowValue(RowIndices[0], "rlnGroupNumber"));
//Star SigmaTable = new Star("D:\\rado27\\Refine3D\\run1_ct5_it009_half1_model.star", "data_model_group_" + GroupNumber);
Star SigmaTable = new Star(MainWindow.Options.ModelStarPath, "data_model_group_" + GroupNumber);
float[] SigmaValues = SigmaTable.GetColumn("rlnSigma2Noise").Select(v => float.Parse(v)).ToArray();
float[] Sigma2NoiseData = Sigma2Noise.GetHost(Intent.Write)[0];
Helper.ForEachElementFT(DimsCropped, (x, y, xx, yy, r, angle) =>
{
int ir = (int)r;
float val = 0;
if (ir < SigmaValues.Length && ir >= size / (4.0f / PixelSize) && ir < DimsCropped.X / 2)
{
if (SigmaValues[ir] != 0f)
val = 1f / SigmaValues[ir] / (ir * 3.14f);
}
Sigma2NoiseData[y * (DimsCropped.X / 2 + 1) + x] = val;
});
float MaxSigma = MathHelper.Max(Sigma2NoiseData);
for (int i = 0; i < Sigma2NoiseData.Length; i++)
Sigma2NoiseData[i] /= MaxSigma;
Sigma2Noise.RemapToFT();
}
//Sigma2Noise.WriteMRC("d_sigma2noiseScore.mrc");
SetPositions(StartParams);
GPU.ProjectForward(VolRefFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Write),
VolRefFT1.Dims,
DimsCropped,
Helper.ToInterleaved(Angles1),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles1 * Dims.Z / 3));
GPU.ProjectForward(VolRefFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Write),
VolRefFT2.Dims,
DimsCropped,
Helper.ToInterleaved(Angles2),
MainWindow.Options.ProjectionOversample,
(uint)(NParticles2 * Dims.Z / 3));
float[] Diff1 = new float[NParticles1];
float[] ParticleQuality1 = new float[NParticles1 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT1.GetDevice(Intent.Read),
Projections1.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams1,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts1.GetDevice(Intent.Read),
Diff1,
ParticleQuality1,
(uint)NParticles1,
(uint)Dims.Z / 3);
float[] Diff2 = new float[NParticles2];
float[] ParticleQuality2 = new float[NParticles2 * (Dims.Z / 3)];
GPU.PolishingGetDiff(ParticleStackFT2.GetDevice(Intent.Read),
Projections2.GetDevice(Intent.Read),
ShiftFactors.GetDevice(Intent.Read),
CTFCoords.GetDevice(Intent.Read),
CTFParams2,
Sigma2Noise.GetDevice(Intent.Read),
DimsCropped,
Shifts2.GetDevice(Intent.Read),
Diff2,
ParticleQuality2,
(uint)NParticles2,
(uint)Dims.Z / 3);
for (int z = 0; z < Dims.Z / 3; z++)
{
for (int p = 0; p < NParticles1; p++)
ParticleQuality[z * NParticles + p] = ParticleQuality1[z * NParticles1 + p];
for (int p = 0; p < NParticles2; p++)
ParticleQuality[z * NParticles + NParticles1 + p] = ParticleQuality2[z * NParticles2 + p];
}
}
#endregion
lock (tableOut) // Only changing cell values, but better be safe in case table implementation changes later
{
GridX = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Take(NParticles * 2).Select(v => (float)v).ToArray());
GridY = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 1).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredX = GridX.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
float[] AlteredY = GridY.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
GridRot = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 2).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridTilt = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 3).Take(NParticles * 2).Select(v => (float)v).ToArray());
GridPsi = new CubicGrid(new int3(NParticles, 1, 2), Optimizer.Solution.Skip(NParticles * 2 * 4).Take(NParticles * 2).Select(v => (float)v).ToArray());
float[] AlteredRot = GridRot.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
float[] AlteredTilt = GridTilt.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
float[] AlteredPsi = GridPsi.GetInterpolated(new int3(NParticles, 1, Dims.Z), new float3(0, 0, 0));
for (int i = 0; i < TableOutIndices.Count; i++)
{
int p = i % NParticles;
int z = i / NParticles;
float Defocus = 0, PhaseShift = 0;
if (p < NParticles1)
{
Defocus = GridCTF.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
PhaseShift = GridCTFPhase.GetInterpolated(new float3((float)Origins1[p].X / Dims.X,
(float)Origins1[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
}
else
{
p -= NParticles1;
Defocus = GridCTF.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
PhaseShift = GridCTFPhase.GetInterpolated(new float3((float)Origins2[p].X / Dims.X,
(float)Origins2[p].Y / Dims.Y,
(float)z / (Dims.Z - 1)));
}
tableOut.SetRowValue(TableOutIndices[i], "rlnOriginX", AlteredX[i].ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnOriginY", AlteredY[i].ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnAngleRot", (-AlteredRot[i]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnAngleTilt", (-AlteredTilt[i]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnAnglePsi", (-AlteredPsi[i]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnDefocusU", ((Defocus + (float)CTF.DefocusDelta / 2f) * 1e4f).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnDefocusV", ((Defocus - (float)CTF.DefocusDelta / 2f) * 1e4f).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnPhaseShift", (PhaseShift * 180f).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnCtfFigureOfMerit", (ParticleQuality[(z / 3) * NParticles + (i % NParticles)]).ToString(CultureInfo.InvariantCulture));
tableOut.SetRowValue(TableOutIndices[i], "rlnMagnification", ((float)MainWindow.Options.CTFDetectorPixel * 10000f / PixelSize).ToString());
}
}
VolRefFT1.Dispose();
VolRefFT2.Dispose();
Projections1.Dispose();
Projections2.Dispose();
Sigma2Noise.Dispose();
ParticleStackFT1.Dispose();
ParticleStackFT2.Dispose();
Shifts1.Dispose();
Shifts2.Dispose();
CTFCoords.Dispose();
ShiftFactors.Dispose();
ParticleStack1.Dispose();
ParticleStack2.Dispose();
PSStack1.Dispose();
PSStack2.Dispose();
}
// Write movies to disk asynchronously, so the next micrograph can load.
Thread SaveThread = new Thread(() =>
{
GPU.SetDevice(CurrentDevice); // It's a separate thread, make sure it's using the same device
ParticleStackAll.WriteMRC(ParticleMoviesPath, ParticlesHeader);
//ParticleStackAll.WriteMRC("D:\\gala\\particlemovies\\" + RootName + "_particles.mrcs", ParticlesHeader);
ParticleStackAll.Dispose();
PSStackAll.WriteMRC(ParticleCTFMoviesPath);
//PSStackAll.WriteMRC("D:\\rado27\\particlectfmovies\\" + RootName + "_particlectf.mrcs");
PSStackAll.Dispose();
});
SaveThread.Start();
}
public void ExportParticles(Star tableIn, Star tableOut, MapHeader originalHeader, Image originalStack, int size, float particleradius, decimal scaleFactor)
{
if (!tableIn.HasColumn("rlnAutopickFigureOfMerit"))
tableIn.AddColumn("rlnAutopickFigureOfMerit");
List<int> RowIndices = new List<int>();
string[] ColumnMicrographName = tableIn.GetColumn("rlnMicrographName");
for (int i = 0; i < ColumnMicrographName.Length; i++)
if (ColumnMicrographName[i].Contains(RootName))
RowIndices.Add(i);
if (RowIndices.Count == 0)
return;
if (!Directory.Exists(ParticlesDir))
Directory.CreateDirectory(ParticlesDir);
if (!Directory.Exists(ParticleCTFDir))
Directory.CreateDirectory(ParticleCTFDir);
int3 Dims = originalHeader.Dimensions;
int3 DimsRegion = new int3(size, size, 1);
int3 DimsPadded = new int3(size * 2, size * 2, 1);
int NParticles = RowIndices.Count;
float PixelSize = (float)CTF.PixelSize;
float PixelDelta = (float)CTF.PixelSizeDelta;
float PixelAngle = (float)CTF.PixelSizeAngle * Helper.ToRad;
/*Image CTFCoords;
Image CTFFreq;
{
float2[] CTFCoordsData = new float2[(DimsRegion.X / 2 + 1) * DimsRegion.Y];
float[] CTFFreqData = new float[(DimsRegion.X / 2 + 1) * DimsRegion.Y];
for (int y = 0; y < DimsRegion.Y; y++)
for (int x = 0; x < DimsRegion.X / 2 + 1; x++)
{
int xx = x;
int yy = y < DimsRegion.Y / 2 + 1 ? y : y - DimsRegion.Y;
float xs = xx / (float)DimsRegion.X;
float ys = yy / (float)DimsRegion.Y;
float r = (float)Math.Sqrt(xs * xs + ys * ys);
float angle = (float)(Math.Atan2(yy, xx));
float CurrentPixelSize = PixelSize + PixelDelta * (float)Math.Cos(2f * (angle - PixelAngle));
CTFCoordsData[y * (DimsRegion.X / 2 + 1) + x] = new float2(r / DimsRegion.X, angle);
CTFFreqData[y * (DimsRegion.X / 2 + 1) + x] = r / CurrentPixelSize;
}
CTFCoords = new Image(CTFCoordsData, DimsRegion.Slice(), true);
CTFFreq = new Image(CTFFreqData, DimsRegion.Slice(), true);
}*/
string[] ColumnPosX = tableIn.GetColumn("rlnCoordinateX");
string[] ColumnPosY = tableIn.GetColumn("rlnCoordinateY");
string[] ColumnOriginX = tableIn.GetColumn("rlnOriginX");
string[] ColumnOriginY = tableIn.GetColumn("rlnOriginY");
int3[] Origins = new int3[NParticles];
float3[] ResidualShifts = new float3[NParticles];
for (int i = 0; i < NParticles; i++)
{
float2 Pos = new float2(float.Parse(ColumnPosX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnPosY[RowIndices[i]], CultureInfo.InvariantCulture)) * 1.00f;
float2 Shift = new float2(float.Parse(ColumnOriginX[RowIndices[i]], CultureInfo.InvariantCulture),
float.Parse(ColumnOriginY[RowIndices[i]], CultureInfo.InvariantCulture)) * 1.00f;
Origins[i] = new int3((int)(Pos.X - Shift.X),
(int)(Pos.Y - Shift.Y),
0);
ResidualShifts[i] = new float3(-MathHelper.ResidualFraction(Pos.X - Shift.X),
-MathHelper.ResidualFraction(Pos.Y - Shift.Y),
0f);
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateX", Origins[i].X.ToString());
tableIn.SetRowValue(RowIndices[i], "rlnCoordinateY", Origins[i].Y.ToString());
tableIn.SetRowValue(RowIndices[i], "rlnOriginX", "0.0");
tableIn.SetRowValue(RowIndices[i], "rlnOriginY", "0.0");
}
Image AverageFT = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true, true);
Image AveragePS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
Image Weights = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
Weights.Fill(1e-6f);
Image FrameParticles = new Image(IntPtr.Zero, new int3(DimsPadded.X, DimsPadded.Y, NParticles));
float StepZ = 1f / Math.Max(Dims.Z - 1, 1);
for (int z = 0; z < Dims.Z; z++)
{
float CoordZ = z * StepZ;
if (originalStack != null)
GPU.Extract(originalStack.GetDeviceSlice(z, Intent.Read),
FrameParticles.GetDevice(Intent.Write),
Dims.Slice(),
DimsPadded,
Helper.ToInterleaved(Origins.Select(v => new int3(v.X - DimsPadded.X / 2, v.Y - DimsPadded.Y / 2, 0)).ToArray()),
(uint)NParticles);
// Shift particles
{
float3[] Shifts = new float3[NParticles];
for (int i = 0; i < NParticles; i++)
{
float3 Coords = new float3((float)Origins[i].X / Dims.X, (float)Origins[i].Y / Dims.Y, CoordZ);
Shifts[i] = ResidualShifts[i] + new float3(GetShiftFromPyramid(Coords)) * 1.00f;
}
FrameParticles.ShiftSlices(Shifts);
}
Image FrameParticlesCropped = FrameParticles.AsPadded(new int2(DimsRegion));
Image FrameParticlesFT = FrameParticlesCropped.AsFFT();
FrameParticlesCropped.Dispose();
//Image PS = new Image(new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
//PS.Fill(1f);
// Apply motion blur filter.
#region Motion blur weighting
/*{
const int Samples = 11;
float StartZ = (z - 0.5f) * StepZ;
float StopZ = (z + 0.5f) * StepZ;
float2[] Shifts = new float2[Samples * NParticles];
for (int p = 0; p < NParticles; p++)
{
float NormX = (float)Origins[p].X / Dims.X;
float NormY = (float)Origins[p].Y / Dims.Y;
for (int zz = 0; zz < Samples; zz++)
{
float zp = StartZ + (StopZ - StartZ) / (Samples - 1) * zz;
float3 Coords = new float3(NormX, NormY, zp);
Shifts[p * Samples + zz] = GetShiftFromPyramid(Coords);
}
}
Image MotionFilter = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
GPU.CreateMotionBlur(MotionFilter.GetDevice(Intent.Write),
DimsRegion,
Helper.ToInterleaved(Shifts.Select(v => new float3(v.X, v.Y, 0)).ToArray()),
Samples,
(uint)NParticles);
PS.Multiply(MotionFilter);
//MotionFilter.WriteMRC("motion.mrc");
MotionFilter.Dispose();
}*/
#endregion
// Apply CTF.
#region CTF weighting
/*if (CTF != null)
{
CTFStruct[] Structs = new CTFStruct[NParticles];
for (int p = 0; p < NParticles; p++)
{
CTF Altered = CTF.GetCopy();
Altered.Defocus = (decimal)GridCTF.GetInterpolated(new float3(Origins[p].X / Dims.X,
Origins[p].Y / Dims.Y,
z * StepZ));
Structs[p] = Altered.ToStruct();
}
Image CTFImage = new Image(IntPtr.Zero, new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
GPU.CreateCTF(CTFImage.GetDevice(Intent.Write),
CTFCoords.GetDevice(Intent.Read),
(uint)CTFCoords.ElementsSliceComplex,
Structs,
false,
(uint)NParticles);
//CTFImage.Abs();
PS.Multiply(CTFImage);
//CTFImage.WriteMRC("ctf.mrc");
CTFImage.Dispose();
}*/
#endregion
// Apply dose.
#region Dose weighting
/*{
float3 NikoConst = new float3(0.245f, -1.665f, 2.81f);
// Niko's formula expects e-/A2/frame, we've got e-/px/frame -- convert!
float FrameDose = (float)MainWindow.Options.CorrectDosePerFrame * (z + 0.5f) / (PixelSize * PixelSize);
Image DoseImage = new Image(IntPtr.Zero, DimsRegion, true);
GPU.DoseWeighting(CTFFreq.GetDevice(Intent.Read),
DoseImage.GetDevice(Intent.Write),
(uint)DoseImage.ElementsSliceComplex,
new[] { FrameDose },
NikoConst,
1);
PS.MultiplySlices(DoseImage);
//DoseImage.WriteMRC("dose.mrc");
DoseImage.Dispose();
}*/
#endregion
//Image PSAbs = new Image(PS.GetDevice(Intent.Read), new int3(DimsRegion.X, DimsRegion.Y, NParticles), true);
//PSAbs.Abs();
//FrameParticlesFT.Multiply(PS);
AverageFT.Add(FrameParticlesFT);
//Weights.Add(PSAbs);
//PS.Multiply(PS);
//AveragePS.Add(PS);
//PS.Dispose();
FrameParticlesFT.Dispose();
//PSAbs.Dispose();
}
FrameParticles.Dispose();
//CTFCoords.Dispose();
//AverageFT.Divide(Weights);
//AveragePS.Divide(Weights);
//AverageFT.Multiply(AveragePS);
Weights.Dispose();
Image AverageParticlesUncorrected = AverageFT.AsIFFT();
AverageFT.Dispose();
Image AverageParticles = AverageParticlesUncorrected.AsAnisotropyCorrected(new int2(DimsRegion),
(float)(CTF.PixelSize + CTF.PixelSizeDelta / 2M),
(float)(CTF.PixelSize - CTF.PixelSizeDelta / 2M),
(float)CTF.PixelSizeAngle * Helper.ToRad,
8);
AverageParticlesUncorrected.Dispose();
GPU.NormParticles(AverageParticles.GetDevice(Intent.Read),
AverageParticles.GetDevice(Intent.Write),
DimsRegion,
(uint)(particleradius / (PixelSize / 1.00f)),
true,
(uint)NParticles);
HeaderMRC ParticlesHeader = new HeaderMRC
{
Pixelsize = new float3(PixelSize, PixelSize, PixelSize)
};
AverageParticles.WriteMRC(ParticlesPath, ParticlesHeader);
AverageParticles.Dispose();
//AveragePS.WriteMRC(ParticleCTFPath, ParticlesHeader);
AveragePS.Dispose();
float[] DistanceWeights = new float[NParticles];
for (int p1 = 0; p1 < NParticles - 1; p1++)
{
float2 Pos1 = new float2(Origins[p1].X, Origins[p1].Y);
for (int p2 = p1 + 1; p2 < NParticles; p2++)
{
float2 Pos2 = new float2(Origins[p2].X, Origins[p2].Y);
float2 Diff = Pos2 - Pos1;
float Dist = Diff.X * Diff.X + Diff.Y * Diff.Y;
Dist = 1f / Dist;
DistanceWeights[p1] += Dist;
DistanceWeights[p2] += Dist;
}
}
for (int i = 0; i < NParticles; i++)
{
string ParticlePath = (i + 1).ToString("D6") + "@particles/" + RootName + "_particles.mrcs";
tableIn.SetRowValue(RowIndices[i], "rlnImageName", ParticlePath);
//string ParticleCTFsPath = (i + 1).ToString("D6") + "@particlectf/" + RootName + "_particlectf.mrcs";
//tableIn.SetRowValue(RowIndices[i], "rlnCtfImage", ParticleCTFsPath);
tableIn.SetRowValue(RowIndices[i], "rlnAutopickFigureOfMerit", DistanceWeights[i].ToString(CultureInfo.InvariantCulture));
}
}
public void UpdateStarDefocus(Star table, string[] columnNames, string[] columnCoordsX, string[] columnCoordsY)
{
List<int> NameIndices = new List<int>();
string InvariantRoot = RootName;
for (int i = 0; i < columnNames.Length; i++)
if (columnNames[i].Contains(InvariantRoot))
NameIndices.Add(i);
//if (NameIndices.Count == 0)
//Debug.WriteLine(RootName + ": " + NameIndices.Count);
foreach (var nameIndex in NameIndices)
{
float CoordX = float.Parse(columnCoordsX[nameIndex], CultureInfo.InvariantCulture);
float CoordY = float.Parse(columnCoordsY[nameIndex], CultureInfo.InvariantCulture);
float LocalDefocus = GridCTF.GetInterpolated(new float3(CoordX / (Dimensions.X * 0.5f), CoordY / (Dimensions.Y * 0.5f), 4f / 37f));
table.SetRowValue(nameIndex, "rlnDefocusU", ((LocalDefocus + (float)CTF.DefocusDelta / 2f) * 1e4f).ToString(CultureInfo.InvariantCulture));
table.SetRowValue(nameIndex, "rlnDefocusV", ((LocalDefocus - (float)CTF.DefocusDelta / 2f) * 1e4f).ToString(CultureInfo.InvariantCulture));
table.SetRowValue(nameIndex, "rlnDefocusAngle", ((float)CTF.DefocusAngle).ToString(CultureInfo.InvariantCulture));
// Go from pixel size to magnification
float Mag = (float)(MainWindow.Options.CTFDetectorPixel * 10000M / (CTF.PixelSize / 1.00M));
table.SetRowValue(nameIndex, "rlnMagnification", Mag.ToString(CultureInfo.InvariantCulture));
//table.SetRowValue(nameIndex, "rlnSphericalAberration", ((float)CTF.Cs).ToString(CultureInfo.InvariantCulture));
//float LocalPhaseShift = GridCTFPhase.GetInterpolated(new float3(0.5f, 0.5f, 4f / 37f));
//table.SetRowValue(nameIndex, "rlnPhaseShift", (LocalPhaseShift * 180f).ToString(CultureInfo.InvariantCulture));
}
}
