public BoxNetTrain(string modelName, Options options)
{
InitializeComponent();
ModelName = modelName;
Options = options;
TextHeader.Text = "Retrain " + ModelName;
TextNewName.Text = ModelName + "_2";
int CorpusSize = 0;
if (Directory.Exists(System.IO.Path.Combine(Environment.CurrentDirectory, "boxnet2training")))
{
foreach (var fileName in Directory.EnumerateFiles(System.IO.Path.Combine(Environment.CurrentDirectory, "boxnet2training"), "*.tif"))
{
MapHeader Header = MapHeader.ReadFromFile(fileName);
CorpusSize += Header.Dimensions.Z / 3;
}
}
CheckCorpus.Content = $" Also use {CorpusSize} micrographs in\n {System.IO.Path.Combine(Environment.CurrentDirectory, "boxnettraining")}";
if (CorpusSize == 0)
{
CheckCorpus.IsChecked = false;
CheckCorpus.IsEnabled = false;
}
}
public static MapHeader LoadHeader(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType)
{
lock (Sync)
{
return(MapHeader.ReadFromFile(path, headerlessSliceDims, headerlessOffset, headerlessType));
}
}
public DialogTomoMatch(TiltSeries[] series, string pathTemplate, Options options)
{
InitializeComponent();
Series = series;
PathTemplate = pathTemplate;
Options = options;
DataContext = Options;
TextTemplateName.Value = Helper.PathToNameWithExtension(pathTemplate);
MapHeader Header = MapHeader.ReadFromFile(pathTemplate);
if (Math.Abs(Header.PixelSize.X - 1f) > 1e-6f)
{
Options.Tasks.TomoMatchTemplatePixel = (decimal)Header.PixelSize.X;
}
}
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, int layer = -1)
{
MapHeader Header = null;
Type ValueType = null;
FileInfo Info = new FileInfo(path);
float[][] Data;
using (BinaryReader Reader = new BinaryReader(OpenWithBigBuffer(path)))
{
Header = MapHeader.ReadFromFile(Reader, Info, headerlessSliceDims, headerlessOffset, headerlessType);
ValueType = Header.GetValueType();
Data = new float[layer < 0 ? Header.Dimensions.Z : 1][];
if (Header.GetType() != typeof(HeaderTiff))
{
for (int z = 0; z < Data.Length; z++)
{
if (layer >= 0)
{
Reader.BaseStream.Seek((int)Header.Dimensions.ElementsSlice() * (int)ImageFormatsHelper.SizeOf(ValueType) * layer, SeekOrigin.Current);
}
byte[] Bytes = Reader.ReadBytes((int)Header.Dimensions.ElementsSlice() * (int)ImageFormatsHelper.SizeOf(ValueType));
Data[z] = new float[(int)Header.Dimensions.ElementsSlice()];
unsafe
{
int Elements = (int)Header.Dimensions.ElementsSlice();
fixed(byte *BytesPtr = Bytes)
fixed(float *DataPtr = Data[z])
{
float *DataP = DataPtr;
if (ValueType == typeof(byte))
{
byte *BytesP = BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(short))
{
short *BytesP = (short *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(ushort))
{
ushort *BytesP = (ushort *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(int))
{
int *BytesP = (int *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(float))
{
float *BytesP = (float *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = *BytesP++;
}
}
else if (ValueType == typeof(double))
{
double *BytesP = (double *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
}
}
}
}
else
{
Data = ((HeaderTiff)Header).ReadData(layer);
}
}
return(Data);
}
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int layer = -1, Stream stream = null, float[][] reuseBuffer = null)
{
MapHeader Header = null;
Type ValueType = null;
float[][] Data;
if (MapHeader.GetHeaderType(path) != typeof(HeaderTiff))
{
using (BinaryReader Reader = isBigEndian ? new BinaryReaderBE(OpenWithBigBuffer(path)) : new BinaryReader(OpenWithBigBuffer(path)))
{
Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType);
ValueType = Header.GetValueType();
Data = reuseBuffer == null ? new float[layer < 0 ? Header.Dimensions.Z : 1][] : reuseBuffer;
int ReadBatchSize = Math.Min((int)Header.Dimensions.ElementsSlice(), 1 << 20);
int ValueSize = (int)ImageFormatsHelper.SizeOf(ValueType);
byte[] Bytes = new byte[ReadBatchSize * ValueSize];
for (int z = 0; z < Data.Length; z++)
{
if (layer >= 0)
{
Reader.BaseStream.Seek(Header.Dimensions.ElementsSlice() * ImageFormatsHelper.SizeOf(ValueType) * layer, SeekOrigin.Current);
}
if (reuseBuffer == null)
{
Data[z] = new float[(int)Header.Dimensions.ElementsSlice()];
}
unsafe
{
fixed(byte *BytesPtr = Bytes)
fixed(float *DataPtr = Data[z])
{
for (int b = 0; b < (int)Header.Dimensions.ElementsSlice(); b += ReadBatchSize)
{
int CurBatch = Math.Min(ReadBatchSize, (int)Header.Dimensions.ElementsSlice() - b);
Reader.Read(Bytes, 0, CurBatch * ValueSize);
if (isBigEndian)
{
if (ValueType == typeof(short) || ValueType == typeof(ushort))
{
for (int i = 0; i < CurBatch * ValueSize / 2; i++)
{
Array.Reverse(Bytes, i * 2, 2);
}
}
else if (ValueType == typeof(int) || ValueType == typeof(float))
{
for (int i = 0; i < CurBatch * ValueSize / 4; i++)
{
Array.Reverse(Bytes, i * 4, 4);
}
}
else if (ValueType == typeof(double))
{
for (int i = 0; i < CurBatch * ValueSize / 8; i++)
{
Array.Reverse(Bytes, i * 8, 8);
}
}
}
float *DataP = DataPtr + b;
if (ValueType == typeof(byte))
{
byte *BytesP = BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(short))
{
short *BytesP = (short *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(ushort))
{
ushort *BytesP = (ushort *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(int))
{
int *BytesP = (int *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(float))
{
float *BytesP = (float *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = *BytesP++;
}
}
else if (ValueType == typeof(double))
{
double *BytesP = (double *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
}
}
}
}
}
}
else
{
Header = MapHeader.ReadFromFile(null, path, headerlessSliceDims, headerlessOffset, headerlessType, stream);
Data = ((HeaderTiff)Header).ReadData(stream, layer);
}
return(Data);
}
public static float[] ReadSmallMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType)
{
FileInfo Info = new FileInfo(path);
float[] Data;
using (BinaryReader Reader = new BinaryReader(OpenWithBigBuffer(path)))
{
MapHeader Header = MapHeader.ReadFromFile(Reader, Info, headerlessSliceDims, headerlessOffset, headerlessType);
Type ValueType = Header.GetValueType();
Data = new float[Header.Dimensions.Elements()];
byte[] Bytes = Reader.ReadBytes((int)Header.Dimensions.Elements() * (int)ImageFormatsHelper.SizeOf(ValueType));
unsafe
{
int Elements = (int)Header.Dimensions.Elements();
fixed(byte *BytesPtr = Bytes)
fixed(float *DataPtr = Data)
{
float *DataP = DataPtr;
if (ValueType == typeof(byte))
{
byte *BytesP = BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(short))
{
short *BytesP = (short *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(ushort))
{
ushort *BytesP = (ushort *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(int))
{
int *BytesP = (int *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(float))
{
float *BytesP = (float *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = *BytesP++;
}
}
else if (ValueType == typeof(double))
{
double *BytesP = (double *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
}
}
}
return(Data);
}
private void PrepareData(string savePath)
{
float Diameter = 1;
Dispatcher.Invoke(() => Diameter = (float)SliderDiameter.Value);
string ErrorString = "";
//if (string.IsNullOrEmpty(NewName))
// ErrorString += "No name specified.\n";
if (string.IsNullOrEmpty(SuffixPositive))
{
ErrorString += "No positive examples selected.\n";
}
if (!string.IsNullOrEmpty(ErrorString))
{
throw new Exception(ErrorString);
}
bool IsNegative = Options.Picking.DataStyle != "cryo";
Movie[] Movies = Options.MainWindow.FileDiscoverer.GetImmutableFiles();
List <Movie> ValidMovies = new List <Movie>();
foreach (var movie in Movies)
{
if (movie.UnselectManual != null && (bool)movie.UnselectManual)
{
continue;
}
if (!File.Exists(movie.AveragePath))
{
continue;
}
bool HasExamples = File.Exists(FolderPositive + movie.RootName + SuffixPositive + ".star");
if (!HasExamples)
{
continue;
}
ValidMovies.Add(movie);
}
if (ValidMovies.Count == 0)
{
throw new Exception("No movie averages could be found to create training examples. Please process the movies first to create the averages.");
}
List <Image> AllAveragesBN = new List <Image>();
List <Image> AllLabelsBN = new List <Image>();
List <Image> AllCertainBN = new List <Image>();
int MoviesDone = 0;
foreach (var movie in ValidMovies)
{
MapHeader Header = MapHeader.ReadFromFile(movie.AveragePath);
float PixelSize = Header.PixelSize.X;
#region Load positions, and possibly move on to next movie
List <float2> PosPositive = new List <float2>();
List <float2> PosFalse = new List <float2>();
List <float2> PosUncertain = new List <float2>();
if (File.Exists(FolderPositive + movie.RootName + SuffixPositive + ".star"))
{
PosPositive.AddRange(Star.LoadFloat2(FolderPositive + movie.RootName + SuffixPositive + ".star",
"rlnCoordinateX",
"rlnCoordinateY").Select(v => v * PixelSize / BoxNet2.PixelSize));
}
//if (PosPositive.Count == 0)
// continue;
if (File.Exists(FolderFalsePositive + movie.RootName + SuffixFalsePositive + ".star"))
{
PosFalse.AddRange(Star.LoadFloat2(FolderFalsePositive + movie.RootName + SuffixFalsePositive + ".star",
"rlnCoordinateX",
"rlnCoordinateY").Select(v => v * PixelSize / BoxNet2.PixelSize));
}
if (File.Exists(FolderUncertain + movie.RootName + SuffixUncertain + ".star"))
{
PosUncertain.AddRange(Star.LoadFloat2(FolderUncertain + movie.RootName + SuffixUncertain + ".star",
"rlnCoordinateX",
"rlnCoordinateY").Select(v => v * PixelSize / BoxNet2.PixelSize));
}
#endregion
Image Average = Image.FromFile(movie.AveragePath);
int2 Dims = new int2(Average.Dims);
Image Mask = null;
if (File.Exists(movie.MaskPath))
{
Mask = Image.FromFile(movie.MaskPath);
}
float RadiusParticle = Math.Max(1, Diameter / 2 / BoxNet2.PixelSize);
float RadiusPeak = Math.Max(1.5f, Diameter / 2 / BoxNet2.PixelSize / 4);
float RadiusFalse = Math.Max(1, Diameter / 2 / BoxNet2.PixelSize);
float RadiusUncertain = Math.Max(1, Diameter / 2 / BoxNet2.PixelSize);
#region Rescale everything and allocate memory
int2 DimsBN = new int2(new float2(Dims) * PixelSize / BoxNet2.PixelSize + 0.5f) / 2 * 2;
Image AverageBN = Average.AsScaled(DimsBN);
Average.Dispose();
if (IsNegative)
{
AverageBN.Multiply(-1f);
}
GPU.Normalize(AverageBN.GetDevice(Intent.Read),
AverageBN.GetDevice(Intent.Write),
(uint)AverageBN.ElementsSliceReal,
1);
Image MaskBN = null;
if (Mask != null)
{
MaskBN = Mask.AsScaled(DimsBN);
Mask.Dispose();
}
Image LabelsBN = new Image(new int3(DimsBN));
Image CertainBN = new Image(new int3(DimsBN));
CertainBN.Fill(1f);
#endregion
#region Paint all positive and uncertain peaks
for (int i = 0; i < 3; i++)
{
var positions = (new[] { PosPositive, PosFalse, PosUncertain })[i];
float R = (new[] { RadiusPeak, RadiusFalse, RadiusUncertain })[i];
float R2 = R * R;
float Label = (new[] { 1, 4, 0 })[i];
float[] ImageData = (new[] { LabelsBN.GetHost(Intent.ReadWrite)[0], CertainBN.GetHost(Intent.ReadWrite)[0], CertainBN.GetHost(Intent.ReadWrite)[0] })[i];
foreach (var pos in positions)
{
int2 Min = new int2(Math.Max(0, (int)(pos.X - R)), Math.Max(0, (int)(pos.Y - R)));
int2 Max = new int2(Math.Min(DimsBN.X - 1, (int)(pos.X + R)), Math.Min(DimsBN.Y - 1, (int)(pos.Y + R)));
for (int y = Min.Y; y <= Max.Y; y++)
{
float yy = y - pos.Y;
yy *= yy;
for (int x = Min.X; x <= Max.X; x++)
{
float xx = x - pos.X;
xx *= xx;
float r2 = xx + yy;
if (r2 <= R2)
{
ImageData[y * DimsBN.X + x] = Label;
}
}
}
}
}
#endregion
#region Add junk mask if there is one
if (MaskBN != null)
{
float[] LabelsBNData = LabelsBN.GetHost(Intent.ReadWrite)[0];
float[] MaskBNData = MaskBN.GetHost(Intent.Read)[0];
for (int i = 0; i < LabelsBNData.Length; i++)
{
if (MaskBNData[i] > 0.5f)
{
LabelsBNData[i] = 2;
}
}
}
#endregion
#region Clean up
MaskBN?.Dispose();
AllAveragesBN.Add(AverageBN);
AverageBN.FreeDevice();
AllLabelsBN.Add(LabelsBN);
LabelsBN.FreeDevice();
AllCertainBN.Add(CertainBN);
CertainBN.FreeDevice();
#endregion
}
#region Figure out smallest dimensions that contain everything
int2 DimsCommon = new int2(1);
foreach (var image in AllAveragesBN)
{
DimsCommon.X = Math.Max(DimsCommon.X, image.Dims.X);
DimsCommon.Y = Math.Max(DimsCommon.Y, image.Dims.Y);
}
#endregion
#region Put everything in one stack and save
Image Everything = new Image(new int3(DimsCommon.X, DimsCommon.Y, AllAveragesBN.Count * 3));
float[][] EverythingData = Everything.GetHost(Intent.ReadWrite);
for (int i = 0; i < AllAveragesBN.Count; i++)
{
if (AllAveragesBN[i].Dims == new int3(DimsCommon)) // No padding needed
{
EverythingData[i * 3 + 0] = AllAveragesBN[i].GetHost(Intent.Read)[0];
EverythingData[i * 3 + 1] = AllLabelsBN[i].GetHost(Intent.Read)[0];
EverythingData[i * 3 + 2] = AllCertainBN[i].GetHost(Intent.Read)[0];
}
else // Padding needed
{
{
Image Padded = AllAveragesBN[i].AsPadded(DimsCommon);
AllAveragesBN[i].Dispose();
EverythingData[i * 3 + 0] = Padded.GetHost(Intent.Read)[0];
Padded.FreeDevice();
}
{
Image Padded = AllLabelsBN[i].AsPadded(DimsCommon);
AllLabelsBN[i].Dispose();
EverythingData[i * 3 + 1] = Padded.GetHost(Intent.Read)[0];
Padded.FreeDevice();
}
{
Image Padded = AllCertainBN[i].AsPadded(DimsCommon);
AllCertainBN[i].Dispose();
EverythingData[i * 3 + 2] = Padded.GetHost(Intent.Read)[0];
Padded.FreeDevice();
}
}
}
Everything.WriteTIFF(savePath, BoxNet2.PixelSize, typeof(float));
#endregion
}
private async void ButtonExport_OnClick(object sender, RoutedEventArgs e)
{
System.Windows.Forms.SaveFileDialog SaveDialog = new System.Windows.Forms.SaveFileDialog
{
Filter = "STAR Files|*.star"
};
System.Windows.Forms.DialogResult ResultSave = SaveDialog.ShowDialog();
if (ResultSave.ToString() == "OK")
{
ExportPath = SaveDialog.FileName;
}
else
{
return;
}
bool DoAverage = (bool)RadioAverage.IsChecked;
bool DoStack = (bool)RadioStack.IsChecked;
bool DoDenoisingPairs = (bool)RadioDenoising.IsChecked;
bool DoOnlyStar = (bool)RadioStar.IsChecked;
bool Invert = (bool)CheckInvert.IsChecked;
bool Normalize = (bool)CheckNormalize.IsChecked;
bool Preflip = (bool)CheckPreflip.IsChecked;
float AngPix = (float)Options.Tasks.InputPixelSize;
bool Relative = (bool)CheckRelative.IsChecked;
bool Filter = (bool)CheckFilter.IsChecked;
bool Manual = (bool)CheckManual.IsChecked;
int BoxSize = (int)Options.Tasks.Export2DBoxSize;
int NormDiameter = (int)Options.Tasks.Export2DParticleDiameter;
ProgressWrite.Visibility = Visibility.Visible;
ProgressWrite.IsIndeterminate = true;
PanelButtons.Visibility = Visibility.Collapsed;
PanelRemaining.Visibility = Visibility.Visible;
foreach (var element in DisableWhileProcessing)
{
element.IsEnabled = false;
}
await Task.Run(async() =>
{
#region Get all movies that can potentially be used
List <Movie> ValidMovies = Movies.Where(v =>
{
if (!Filter && v.UnselectFilter && v.UnselectManual == null)
{
return(false);
}
if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual)
{
return(false);
}
if (v.OptionsCTF == null)
{
return(false);
}
return(true);
}).ToList();
List <string> ValidMovieNames = ValidMovies.Select(m => m.RootName).ToList();
if (ValidMovies.Count == 0)
{
await Dispatcher.Invoke(async() =>
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
"No items were found to extract particles from.\n" +
"Please make sure the names match, estimate the CTF for all items, and review your filtering thresholds.");
});
}
#endregion
#region Read table and intersect its micrograph set with valid movies
Star TableIn;
if (Options.Tasks.InputOnePerItem)
{
List <Star> Tables = new List <Star>();
foreach (var item in Movies)
{
string StarPath = InputFolder + item.RootName + InputSuffix + ".star";
if (File.Exists(StarPath))
{
Star TableItem = new Star(StarPath);
if (!TableItem.HasColumn("rlnMicrographName"))
{
TableItem.AddColumn("rlnMicrographName", item.Name);
}
if (item.PickingThresholds.ContainsKey(InputSuffix) && TableItem.HasColumn("rlnAutopickFigureOfMerit"))
{
float Threshold = (float)item.PickingThresholds[InputSuffix];
float[] Scores = TableItem.GetColumn("rlnAutopickFigureOfMerit").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
int[] InvalidRows = Helper.ArrayOfSequence(0, TableItem.RowCount, 1).Where(i => Scores[i] < Threshold).ToArray();
TableItem.RemoveRows(InvalidRows);
}
Tables.Add(TableItem);
}
}
TableIn = new Star(Tables.ToArray());
}
else
{
TableIn = new Star(ImportPath);
}
if (!TableIn.HasColumn("rlnMicrographName"))
{
throw new Exception("Couldn't find rlnMicrographName column.");
}
if (!TableIn.HasColumn("rlnCoordinateX"))
{
throw new Exception("Couldn't find rlnCoordinateX column.");
}
if (!TableIn.HasColumn("rlnCoordinateY"))
{
throw new Exception("Couldn't find rlnCoordinateY column.");
}
Dictionary <string, List <int> > Groups = new Dictionary <string, List <int> >();
{
string[] ColumnMicNames = TableIn.GetColumn("rlnMicrographName");
for (int r = 0; r < ColumnMicNames.Length; r++)
{
if (!Groups.ContainsKey(ColumnMicNames[r]))
{
Groups.Add(ColumnMicNames[r], new List <int>());
}
Groups[ColumnMicNames[r]].Add(r);
}
Groups = Groups.ToDictionary(group => Helper.PathToName(group.Key), group => group.Value);
Groups = Groups.Where(group => ValidMovieNames.Contains(group.Key)).ToDictionary(group => group.Key, group => group.Value);
}
bool[] RowsIncluded = new bool[TableIn.RowCount];
foreach (var group in Groups)
{
foreach (var r in group.Value)
{
RowsIncluded[r] = true;
}
}
List <int> RowsNotIncluded = new List <int>();
for (int r = 0; r < RowsIncluded.Length; r++)
{
if (!RowsIncluded[r])
{
RowsNotIncluded.Add(r);
}
}
ValidMovies = ValidMovies.Where(v => Groups.ContainsKey(v.RootName)).ToList();
if (ValidMovies.Count == 0) // Exit if there is nothing to export, otherwise errors will be thrown below
{
return;
}
#endregion
#region Make sure all columns are there
if (!TableIn.HasColumn("rlnMagnification"))
{
TableIn.AddColumn("rlnMagnification", "10000.0");
}
else
{
TableIn.SetColumn("rlnMagnification", Helper.ArrayOfConstant("10000.0", TableIn.RowCount));
}
if (!TableIn.HasColumn("rlnDetectorPixelSize"))
{
TableIn.AddColumn("rlnDetectorPixelSize", Options.GetProcessingParticleExport().BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture));
}
else
{
TableIn.SetColumn("rlnDetectorPixelSize", Helper.ArrayOfConstant(Options.GetProcessingParticleExport().BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture), TableIn.RowCount));
}
if (!TableIn.HasColumn("rlnVoltage"))
{
TableIn.AddColumn("rlnVoltage", "300.0");
}
if (!TableIn.HasColumn("rlnSphericalAberration"))
{
TableIn.AddColumn("rlnSphericalAberration", "2.7");
}
if (!TableIn.HasColumn("rlnAmplitudeContrast"))
{
TableIn.AddColumn("rlnAmplitudeContrast", "0.07");
}
if (!TableIn.HasColumn("rlnPhaseShift"))
{
TableIn.AddColumn("rlnPhaseShift", "0.0");
}
if (!TableIn.HasColumn("rlnDefocusU"))
{
TableIn.AddColumn("rlnDefocusU", "0.0");
}
if (!TableIn.HasColumn("rlnDefocusV"))
{
TableIn.AddColumn("rlnDefocusV", "0.0");
}
if (!TableIn.HasColumn("rlnDefocusAngle"))
{
TableIn.AddColumn("rlnDefocusAngle", "0.0");
}
if (!TableIn.HasColumn("rlnCtfMaxResolution"))
{
TableIn.AddColumn("rlnCtfMaxResolution", "999.0");
}
if (!TableIn.HasColumn("rlnImageName"))
{
TableIn.AddColumn("rlnImageName", "None");
}
if (!TableIn.HasColumn("rlnMicrographName"))
{
TableIn.AddColumn("rlnMicrographName", "None");
}
#endregion
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
List <int> UsedDeviceProcesses = Helper.Combine(Helper.ArrayOfFunction(i => UsedDevices.Select(d => d + i *NDevices).ToArray(), MainWindow.GlobalOptions.ProcessesPerDevice)).ToList();
if (IsCanceled)
{
return;
}
#region Create worker processes
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount() * MainWindow.GlobalOptions.ProcessesPerDevice];
foreach (var gpuID in UsedDeviceProcesses)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(Options.Import.HeaderlessWidth, Options.Import.HeaderlessHeight),
Options.Import.HeaderlessOffset,
Options.Import.HeaderlessType);
if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain)
{
Workers[gpuID].LoadGainRef(Options.Import.CorrectGain ? Options.Import.GainPath : "",
Options.Import.GainFlipX,
Options.Import.GainFlipY,
Options.Import.GainTranspose,
Options.Import.CorrectDefects ? Options.Import.DefectsPath : "");
}
}
#endregion
#region Load gain reference if needed
//Image[] ImageGain = new Image[NDevices];
//if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain && File.Exists(Options.Import.GainPath))
// for (int d = 0; d < NDevices; d++)
// {
// GPU.SetDevice(d);
// ImageGain[d] = MainWindow.LoadAndPrepareGainReference();
// }
#endregion
bool Overwrite = true;
if (DoAverage || DoStack)
{
foreach (var movie in ValidMovies)
{
bool FileExists = File.Exists((DoAverage ? movie.ParticlesDir : movie.ParticleMoviesDir) + movie.RootName + Options.Tasks.OutputSuffix + ".mrcs");
if (FileExists)
{
await Dispatcher.Invoke(async() =>
{
var DialogResult = await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Some particle files already exist. Overwrite them?",
"",
MessageDialogStyle.AffirmativeAndNegative,
new MetroDialogSettings()
{
AffirmativeButtonText = "Yes",
NegativeButtonText = "No"
});
if (DialogResult == MessageDialogResult.Negative)
{
Overwrite = false;
}
});
break;
}
}
}
Star TableOut = null;
{
Dictionary <string, Star> MicrographTables = new Dictionary <string, Star>();
#region Get coordinates
float[] PosX = TableIn.GetColumn("rlnCoordinateX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] PosY = TableIn.GetColumn("rlnCoordinateY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] ShiftX = TableIn.HasColumn("rlnOriginX") ? TableIn.GetColumn("rlnOriginX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
float[] ShiftY = TableIn.HasColumn("rlnOriginY") ? TableIn.GetColumn("rlnOriginY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
for (int r = 0; r < TableIn.RowCount; r++)
{
PosX[r] -= ShiftX[r];
PosY[r] -= ShiftY[r];
}
if (TableIn.HasColumn("rlnOriginX"))
{
TableIn.RemoveColumn("rlnOriginX");
}
if (TableIn.HasColumn("rlnOriginY"))
{
TableIn.RemoveColumn("rlnOriginY");
}
#endregion
Dispatcher.Invoke(() => ProgressWrite.MaxValue = ValidMovies.Count);
Helper.ForEachGPU(ValidMovies, (movie, gpuID) =>
{
if (IsCanceled)
{
return;
}
Stopwatch ItemTime = new Stopwatch();
ItemTime.Start();
MapHeader OriginalHeader = MapHeader.ReadFromFile(movie.Path);
#region Set up export options
ProcessingOptionsParticlesExport ExportOptions = Options.GetProcessingParticleExport();
ExportOptions.DoAverage = DoAverage;
ExportOptions.DoDenoisingPairs = DoDenoisingPairs;
ExportOptions.DoStack = DoStack;
ExportOptions.Invert = Invert;
ExportOptions.Normalize = Normalize;
ExportOptions.PreflipPhases = Preflip;
ExportOptions.Dimensions = OriginalHeader.Dimensions.MultXY((float)Options.PixelSizeMean);
ExportOptions.BoxSize = BoxSize;
ExportOptions.Diameter = NormDiameter;
#endregion
bool FileExists = File.Exists((DoAverage ? movie.ParticlesDir : movie.ParticleMoviesDir) + movie.RootName + ExportOptions.Suffix + ".mrcs");
#region Load and prepare original movie
//Image OriginalStack = null;
decimal ScaleFactor = 1M / (decimal)Math.Pow(2, (double)ExportOptions.BinTimes);
if (!DoOnlyStar && (!FileExists || Overwrite))
{
Workers[gpuID].LoadStack(movie.Path, ScaleFactor, ExportOptions.EERGroupFrames);
}
//MainWindow.LoadAndPrepareHeaderAndMap(movie.Path, ImageGain[gpuID], ScaleFactor, out OriginalHeader, out OriginalStack);
if (IsCanceled)
{
//foreach (Image gain in ImageGain)
// gain?.Dispose();
//OriginalStack?.Dispose();
return;
}
#endregion
#region Figure out relative or absolute path to particle stack
string PathStack = (ExportOptions.DoStack ? movie.ParticleMoviesDir : movie.ParticlesDir) + movie.RootName + ExportOptions.Suffix + ".mrcs";
string PathMicrograph = movie.Path;
if (Relative)
{
Uri UriStar = new Uri(ExportPath);
PathStack = UriStar.MakeRelativeUri(new Uri(PathStack)).ToString();
PathMicrograph = UriStar.MakeRelativeUri(new Uri(PathMicrograph)).ToString();
}
#endregion
#region Update row values
List <int> GroupRows = Groups[movie.RootName];
List <float2> Positions = new List <float2>();
float Astigmatism = (float)movie.CTF.DefocusDelta / 2;
float PhaseShift = movie.OptionsCTF.DoPhase ? movie.GridCTFPhase.GetInterpolated(new float3(0.5f)) * 180 : 0;
int ImageNameIndex = TableIn.GetColumnID("rlnImageName");
foreach (var r in GroupRows)
{
float3 Position = new float3(PosX[r] * AngPix / ExportOptions.Dimensions.X,
PosY[r] * AngPix / ExportOptions.Dimensions.Y,
0.5f);
float LocalDefocus = movie.GridCTFDefocus.GetInterpolated(Position);
TableIn.SetRowValue(r, "rlnDefocusU", ((LocalDefocus + Astigmatism) * 1e4f).ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnDefocusV", ((LocalDefocus - Astigmatism) * 1e4f).ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnDefocusAngle", movie.CTF.DefocusAngle.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnVoltage", movie.CTF.Voltage.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnSphericalAberration", movie.CTF.Cs.ToString("F4", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnAmplitudeContrast", movie.CTF.Amplitude.ToString("F3", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnPhaseShift", PhaseShift.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCtfMaxResolution", movie.CTFResolutionEstimate.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateX", (PosX[r] * AngPix / (float)ExportOptions.BinnedPixelSizeMean).ToString("F2", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateY", (PosY[r] * AngPix / (float)ExportOptions.BinnedPixelSizeMean).ToString("F2", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnImageName", PathStack);
TableIn.SetRowValue(r, "rlnMicrographName", PathMicrograph);
Positions.Add(new float2(PosX[r] * AngPix,
PosY[r] * AngPix));
}
#endregion
#region Populate micrograph table with rows for all exported particles
Star MicrographTable = new Star(TableIn.GetColumnNames());
int StackDepth = (ExportOptions.DoAverage || ExportOptions.DoDenoisingPairs || DoOnlyStar)
? 1
: (OriginalHeader.Dimensions.Z - ExportOptions.SkipFirstN - ExportOptions.SkipLastN) /
ExportOptions.StackGroupSize;
int pi = 0;
for (int i = 0; i < StackDepth; i++)
{
foreach (var r in GroupRows)
{
List <string> Row = TableIn.GetRow(r).ToList();
Row[ImageNameIndex] = (++pi).ToString("D7") + "@" + Row[ImageNameIndex];
MicrographTable.AddRow(Row);
}
}
#endregion
#region Finally, process and export the actual particles
if (!DoOnlyStar && (!FileExists || Overwrite))
{
Workers[gpuID].MovieExportParticles(movie.Path, ExportOptions, Positions.ToArray());
//movie.ExportParticles(OriginalStack, Positions.ToArray(), ExportOptions);
//OriginalStack.Dispose();
}
#endregion
#region Add this micrograph's table to global collection, update remaining time estimate
lock (MicrographTables)
{
MicrographTables.Add(movie.RootName, MicrographTable);
Timings.Add(ItemTime.ElapsedMilliseconds / (float)NDevices);
int MsRemaining = (int)(MathHelper.Mean(Timings) * (ValidMovies.Count - MicrographTables.Count));
TimeSpan SpanRemaining = new TimeSpan(0, 0, 0, 0, MsRemaining);
Dispatcher.Invoke(() => TextRemaining.Text = SpanRemaining.ToString((int)SpanRemaining.TotalHours > 0 ? @"hh\:mm\:ss" : @"mm\:ss"));
Dispatcher.Invoke(() =>
{
ProgressWrite.IsIndeterminate = false;
ProgressWrite.Value = MicrographTables.Count;
});
}
#endregion
}, 1, UsedDeviceProcesses);
if (MicrographTables.Count > 0)
{
TableOut = new Star(MicrographTables.Values.ToArray());
}
}
Thread.Sleep(10000); // Writing out particle stacks is async, so if workers are killed immediately they may not write out everything
foreach (var worker in Workers)
{
worker?.Dispose();
}
//foreach (Image gain in ImageGain)
// gain?.Dispose();
if (IsCanceled)
{
return;
}
TableOut.Save(ExportPath);
});
DataContext = null;
Close?.Invoke();
}
private async void ButtonAddLocal_OnClick(object sender, RoutedEventArgs e)
{
System.Windows.Forms.OpenFileDialog OpenDialog = new System.Windows.Forms.OpenFileDialog();
OpenDialog.Filter = "Warp Folder Settings|*.settings|Warp Data Source|*.source";
System.Windows.Forms.DialogResult OpenResult = OpenDialog.ShowDialog();
if (OpenResult.ToString() == "OK")
{
FileInfo Info = new FileInfo(OpenDialog.FileName);
#region Check if user wants to use an existing source file instead
//if (Info.Extension.ToLower() == ".settings" &&
// File.Exists(OpenDialog.FileName.Substring(0, OpenDialog.FileName.LastIndexOf(".")) + ".source"))
//{
// var Result = await ((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Similar source file found",
// $"Would you like to use {Helper.PathToName(Info.Name) + ".source"} instead?",
// MessageDialogStyle.AffirmativeAndNegativeAndSingleAuxiliary,
// new MetroDialogSettings
// {
// AffirmativeButtonText = "Use existing source",
// NegativeButtonText = "Replace with new source from settings",
// FirstAuxiliaryButtonText = "Cancel"
// });
// if (Result == MessageDialogResult.FirstAuxiliary) // Cancel
// {
// return;
// }
// else if (Result == MessageDialogResult.Affirmative) // Use existing .source
// {
// OpenDialog.FileName = OpenDialog.FileName.Substring(0, OpenDialog.FileName.LastIndexOf(".")) + ".source";
// Info = new FileInfo(OpenDialog.FileName);
// }
//}
#endregion
if (Info.Extension.ToLower() == ".settings")
{
#region Load preprocessing options
Warp.Options Options = new Warp.Options();
try
{
Options.Load(OpenDialog.FileName);
}
catch
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
$"An error was encountered when reading {Info.Name}.",
MessageDialogStyle.Affirmative);
return;
}
#endregion
#region Load items with metadata
string FileExtension = Options.Import.Extension;
var AvailableFiles = Directory.EnumerateFiles(Info.DirectoryName, FileExtension).ToArray();
Movie[] Items = new Movie[AvailableFiles.Length];
string[] MatchingStarNames = null;
if (FileExtension != "*.tomostar" && AvailableFiles.Length > 0)
{
Movie First = new Movie(AvailableFiles[0]);
MatchingStarNames = Directory.EnumerateFiles(First.MatchingDir, "*.star").ToArray();
}
{
var ProgressDialog = await((MainWindow)Application.Current.MainWindow).ShowProgressAsync("Loading metadata...", "");
ProgressDialog.Maximum = AvailableFiles.Count();
await Task.Run(() =>
{
int Done = 0;
Parallel.For(0, AvailableFiles.Length, i =>
{
string file = AvailableFiles[i];
string XmlPath = file.Substring(0, file.LastIndexOf(".")) + ".xml";
if (File.Exists(XmlPath))
{
Items[i] = (FileExtension == "*.tomostar" ? new TiltSeries(file) : new Movie(file, MatchingStarNames));
}
lock (Items)
ProgressDialog.SetProgress(++Done);
});
});
await ProgressDialog.CloseAsync();
}
if (Items.Length == 0)
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
$"No micrographs or tilt series found to match these settings.",
MessageDialogStyle.Affirmative);
return;
}
#endregion
#region Figure out filtering status
#region Astigmatism statistics
Movie[] ItemsWithCTF = Items.Where(v => v.OptionsCTF != null && v.CTF != null).ToArray();
List <float2> AstigmatismPoints = new List <float2>(ItemsWithCTF.Length);
foreach (var item in ItemsWithCTF)
{
AstigmatismPoints.Add(new float2((float)Math.Cos((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta,
(float)Math.Sin((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta));
}
float2 AstigmatismMean = new float2();
float AstigmatismStd = 0.1f;
if (AstigmatismPoints.Count > 0)
{
AstigmatismMean = new float2();
foreach (var point in AstigmatismPoints)
{
AstigmatismMean += point;
}
AstigmatismMean /= AstigmatismPoints.Count;
AstigmatismStd = 0;
foreach (var point in AstigmatismPoints)
{
AstigmatismStd += (point - AstigmatismMean).LengthSq();
}
AstigmatismStd = (float)Math.Max(1e-4, Math.Sqrt(AstigmatismStd / AstigmatismPoints.Count));
}
#endregion
int[] ParticleCounts = Items.Select(item => item.GetParticleCount(Options.Filter.ParticlesSuffix)).ToArray();
bool HaveParticles = ParticleCounts.Any(v => v > 0);
foreach (var item in Items)
{
bool FilterStatus = true;
if (item.OptionsCTF != null)
{
FilterStatus &= item.CTF.Defocus >= Options.Filter.DefocusMin && item.CTF.Defocus <= Options.Filter.DefocusMax;
float AstigmatismDeviation = (new float2((float)Math.Cos((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta,
(float)Math.Sin((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta) - AstigmatismMean).Length() / AstigmatismStd;
FilterStatus &= AstigmatismDeviation <= (float)Options.Filter.AstigmatismMax;
FilterStatus &= item.CTFResolutionEstimate <= Options.Filter.ResolutionMax;
FilterStatus &= item.CTF.PhaseShift >= Options.Filter.PhaseMin && item.CTF.PhaseShift <= Options.Filter.PhaseMax;
FilterStatus &= item.GetParticleCount(Options.Filter.ParticlesSuffix) >= Options.Filter.ParticlesMin;
}
if (item.OptionsMovement != null)
{
FilterStatus &= item.MeanFrameMovement <= Options.Filter.MotionMax;
}
item.UnselectFilter = !FilterStatus;
}
ProcessingOptionsMovieCTF OptionsCTF = Options.GetProcessingMovieCTF();
ProcessingOptionsMovieMovement OptionsMovement = Options.GetProcessingMovieMovement();
ProcessingOptionsBoxNet OptionsBoxNet = Options.GetProcessingBoxNet();
ProcessingOptionsMovieExport OptionsExport = Options.GetProcessingMovieExport();
List <Movie> ItemsProcessed = new List <Movie>();
List <Movie> ItemsFilteredOut = new List <Movie>();
List <Movie> ItemsUnselected = new List <Movie>();
foreach (Movie item in Items)
{
ProcessingStatus Status = GetMovieProcessingStatus(item, OptionsCTF, OptionsMovement, OptionsBoxNet, OptionsExport, Options);
if (Status == ProcessingStatus.Processed || (Status == ProcessingStatus.Outdated && !item.UnselectFilter))
{
ItemsProcessed.Add(item);
}
else if (Status == ProcessingStatus.FilteredOut || (Status == ProcessingStatus.Outdated && item.UnselectFilter))
{
ItemsFilteredOut.Add(item);
}
else if (Status == ProcessingStatus.LeaveOut)
{
ItemsUnselected.Add(item);
}
}
#endregion
#region Figure out how many frames/tilts there are to use
int UsableFrames = 1;
if (Items[0].GetType() == typeof(Movie))
{
UsableFrames = MapHeader.ReadFromFile(Items[0].Path).Dimensions.Z;
}
else
{
UsableFrames = ((TiltSeries)Items[0]).NTilts;
}
#endregion
#region Show dialog
CustomDialog Dialog = new CustomDialog();
Dialog.HorizontalContentAlignment = HorizontalAlignment.Center;
DialogCreateSourceFromSettings DialogContent = new DialogCreateSourceFromSettings();
DialogContent.TextTitle.Text = $"Create data source from\n{Info.Name}";
DialogContent.StatsSeriesStatusProcessed.Values = new ChartValues <ObservableValue> {
new ObservableValue(ItemsProcessed.Count)
};
DialogContent.StatsSeriesStatusUnfiltered.Values = new ChartValues <ObservableValue> {
new ObservableValue(ItemsFilteredOut.Count)
};
DialogContent.StatsSeriesStatusUnselected.Values = new ChartValues <ObservableValue> {
new ObservableValue(ItemsUnselected.Count)
};
DialogContent.SliderFrames.Value = UsableFrames;
DialogContent.SliderFrames.MaxValue = UsableFrames;
DialogContent.Close += () =>
{
((MainWindow)Application.Current.MainWindow).HideMetroDialogAsync(Dialog);
};
DialogContent.Create += async() =>
{
#region Create source metadata and check if one with the same path already exists
DataSource NewSource = new DataSource
{
PixelSizeX = Options.PixelSizeX,
PixelSizeY = Options.PixelSizeY,
PixelSizeAngle = Options.PixelSizeAngle,
DimensionsX = Options.Tomo.DimensionsX,
DimensionsY = Options.Tomo.DimensionsY,
DimensionsZ = Options.Tomo.DimensionsZ,
FrameLimit = (int)DialogContent.SliderFrames.Value,
GainPath = Options.Import.CorrectGain ? Options.Import.GainPath : "",
GainFlipX = Options.Import.GainFlipX,
GainFlipY = Options.Import.GainFlipY,
GainTranspose = Options.Import.GainTranspose,
DosePerAngstromFrame = Options.Import.DosePerAngstromFrame,
Name = DialogContent.TextSourceName.Text,
Path = Info.DirectoryName + "\\" + Helper.RemoveInvalidChars(DialogContent.TextSourceName.Text) + ".source"
};
if (Population.Sources.Any(s => s.Path == NewSource.Path))
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
$"{Helper.PathToNameWithExtension(NewSource.Path)} already exists in this population. Please choose a different name.",
MessageDialogStyle.Affirmative);
return;
}
await((MainWindow)Application.Current.MainWindow).HideMetroDialogAsync(Dialog);
#endregion
#region Add all items and their data hashes
List <Movie> AllItems = new List <Movie>(ItemsProcessed);
if ((bool)DialogContent.CheckFilter.IsChecked)
{
AllItems.AddRange(ItemsFilteredOut);
}
if ((bool)DialogContent.CheckManual.IsChecked)
{
AllItems.AddRange(ItemsUnselected);
}
if (AllItems.Count == 0)
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
$"No micrographs or tilt series found to match these settings.",
MessageDialogStyle.Affirmative);
return;
}
{
var ProgressDialog = await((MainWindow)Application.Current.MainWindow).ShowProgressAsync("Calculating data hashes...", "");
ProgressDialog.Maximum = AllItems.Count;
await Task.Run(() =>
{
int Done = 0;
foreach (var item in AllItems)
{
NewSource.Files.Add(item.GetDataHash(), item.Name);
ProgressDialog.SetProgress(++Done);
}
});
await ProgressDialog.CloseAsync();
}
#endregion
#region Check for overlapping hashes
string[] Overlapping = Helper.Combine(Population.Sources.Select(s => s.Files.Where(f => NewSource.Files.ContainsKey(f.Key)).Select(f => f.Value).ToArray()));
if (Overlapping.Length > 0)
{
string Offenders = "";
for (int o = 0; o < Math.Min(5, Overlapping.Length); o++)
{
Offenders += "\n" + Overlapping[o];
}
if (Overlapping.Length > 5)
{
Offenders += $"\n... and {Overlapping.Length - 5} more.";
}
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
"The new source contains files that are already used in this population:" + Offenders,
MessageDialogStyle.Affirmative);
return;
}
#endregion
{
var ProgressDialog = await((MainWindow)Application.Current.MainWindow).ShowProgressAsync("Committing initial version...", "");
ProgressDialog.Maximum = AllItems.Count;
await Task.Run(() =>
{
NewSource.Commit();
});
await ProgressDialog.CloseAsync();
}
Population.Sources.Add(NewSource);
UpdateGridItems();
};
Dialog.Content = DialogContent;
await((MainWindow)Application.Current.MainWindow).ShowMetroDialogAsync(Dialog, new MetroDialogSettings()
{
});
#endregion
}
else
{
try
{
if (Population.Sources.Any(s => s.Path == OpenDialog.FileName))
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
"This data source is already part of this population.",
MessageDialogStyle.Affirmative);
return;
}
DataSource NewSource = DataSource.FromFile(OpenDialog.FileName);
Population.Sources.Add(NewSource);
UpdateGridItems();
}
catch
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
$"An error was encountered when reading {Info.Name}.",
MessageDialogStyle.Affirmative);
return;
}
}
}
}
private async void ButtonWrite_OnClick(object sender, RoutedEventArgs e)
{
string Suffix = TextSuffix.Text;
float ScaleFactorCoords = (float)(Options.Tasks.InputPixelSize / Options.BinnedPixelSizeMean);
float ScaleFactorShifts = (float)(Options.Tasks.InputShiftPixelSize / Options.BinnedPixelSizeMean);
ProgressWrite.Visibility = Visibility.Visible;
PanelButtons.Visibility = Visibility.Hidden;
await Task.Run(async() =>
{
Star TableIn = new Star(ImportPath);
#region Make sure everything is there
if (TableIn.RowCount == 0 || !TableIn.HasColumn("rlnCoordinateX") || !TableIn.HasColumn("rlnCoordinateY") || !TableIn.HasColumn("rlnMicrographName"))
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
"Please make sure the table contains data with rlnCoordinateX, rlnCoordinateY and rlnMicrographName columns.");
Close?.Invoke();
return;
}
if (Movies.Length == 0)
{
Close?.Invoke();
return;
}
#endregion
Directory.CreateDirectory(Movies[0].MatchingDir);
Dispatcher.Invoke(() => ProgressWrite.Maximum = Movies.Length);
int IndexMicName = TableIn.GetColumnID("rlnMicrographName");
TableIn.ModifyAllValuesInColumn("rlnMicrographName", v => Helper.PathToName(v));
int IndexCoordX = TableIn.GetColumnID("rlnCoordinateX");
int IndexCoordY = TableIn.GetColumnID("rlnCoordinateY");
#region Subtract origin from coordinates
int IndexOriginX = TableIn.GetColumnID("rlnOriginX");
if (IndexOriginX >= 0)
{
TableIn.ModifyAllValuesInColumn("rlnCoordinateX", (v, r) => (float.Parse(v) * ScaleFactorCoords - float.Parse(TableIn.GetRowValue(r, IndexOriginX)) * ScaleFactorShifts).ToString("F4"));
TableIn.SetColumn("rlnOriginX", Helper.ArrayOfConstant("0.0", TableIn.RowCount));
}
else
{
TableIn.ModifyAllValuesInColumn("rlnCoordinateX", v => (float.Parse(v) * ScaleFactorCoords).ToString("F4"));
}
int IndexOriginY = TableIn.GetColumnID("rlnOriginY");
if (IndexOriginY >= 0)
{
TableIn.ModifyAllValuesInColumn("rlnCoordinateY", (v, r) => (float.Parse(v) * ScaleFactorCoords - float.Parse(TableIn.GetRowValue(r, IndexOriginY)) * ScaleFactorShifts).ToString("F4"));
TableIn.SetColumn("rlnOriginY", Helper.ArrayOfConstant("0.0", TableIn.RowCount));
}
else
{
TableIn.ModifyAllValuesInColumn("rlnCoordinateY", v => (float.Parse(v) * ScaleFactorCoords).ToString("F4"));
}
#endregion
var GroupedRows = TableIn.GetAllRows().GroupBy(row => row[IndexMicName]).ToDictionary(g => g.Key, g => g.ToList());
foreach (var movie in Movies)
{
if (GroupedRows.ContainsKey(movie.RootName))
{
int3 Dims = MapHeader.ReadFromFile(movie.Path,
new int2(Options.Import.HeaderlessWidth, Options.Import.HeaderlessHeight),
Options.Import.HeaderlessOffset,
ImageFormatsHelper.StringToType(Options.Import.HeaderlessType)).Dimensions;
float3 BinnedDims = new float3(Dims) * (float)Options.PixelSizeMean / (float)Options.BinnedPixelSizeMean;
List <List <string> > Rows = GroupedRows[movie.RootName];
foreach (var row in Rows)
{
row[IndexMicName] = movie.Name;
float CoordX = float.Parse(row[IndexCoordX]);
if (Options.Tasks.InputFlipX)
{
CoordX = BinnedDims.X - CoordX;
}
float CoordY = float.Parse(row[IndexCoordY]);
if (Options.Tasks.InputFlipY)
{
CoordY = BinnedDims.Y - CoordY;
}
row[IndexCoordX] = CoordX.ToString("F4");
row[IndexCoordY] = CoordY.ToString("F4");
}
Star TableOut = new Star(TableIn.GetColumnNames());
foreach (var row in Rows)
{
TableOut.AddRow(row);
}
TableOut.Save(movie.MatchingDir + movie.RootName + "_" + Suffix + ".star");
movie.UpdateParticleCount("_" + Suffix);
}
Dispatcher.Invoke(() => ProgressWrite.Value++);
}
});
Close?.Invoke();
}
private async void ButtonCreateStacks_Click(object sender, RoutedEventArgs e)
{
var ProgressDialog = await Options.MainWindow.ShowProgressAsync("Writing data...", "");
ObservableCollection <ParsedEntry> _ParsedEntries = new ObservableCollection <ParsedEntry>(ParsedEntries.Where(v => v.DoImport));
await Task.Run(() =>
{
Image StackBuffer = null;
int MaxTilts = 0;
int idone = 0;
foreach (var entry in _ParsedEntries)
{
Dispatcher.Invoke(() => ProgressDialog.SetTitle($"Writing stack for {entry.Name}..."));
Directory.CreateDirectory(Path.Combine(Options.Import.Folder, "imod", entry.Name));
Movie[] Movies = Helper.ArrayOfFunction(i => new Movie(Path.Combine(Options.Import.Folder, entry.Table.GetRowValue(i, "wrpMovieName"))), entry.Table.RowCount);
MapHeader Header = MapHeader.ReadFromFile(Movies[0].AveragePath);
int3 StackDims = new int3(Header.Dimensions.X, Header.Dimensions.Y, Movies.Length);
if (StackBuffer == null)
{
StackBuffer = new Image(StackDims);
}
else if (StackBuffer.Dims != StackDims)
{
if (MaxTilts < StackDims.Z || StackBuffer.Dims.Slice() != StackDims.Slice())
{
StackBuffer.Dispose();
StackBuffer = new Image(StackDims);
}
else
{
StackBuffer.Dims = StackDims;
}
}
MaxTilts = StackDims.Z;
float[][] StackData = StackBuffer.GetHost(Intent.Write);
for (int i = 0; i < Movies.Length; i++)
{
IOHelper.ReadMapFloatPatient(50, 500,
Movies[i].AveragePath,
new int2(1),
0,
typeof(float),
new[] { 0 },
null,
new[] { StackData[i] });
}
//HeaderMRC StackHeader = new HeaderMRC();
//StackHeader.ImodTilt = entry.TiltAngles;
//StackHeader.ImodRotation = entry.Rotation;
//StackHeader.PixelSize = Header.PixelSize;
StackBuffer.WriteMRC(Path.Combine(Options.Import.Folder, "imod", entry.Name, entry.Name + ".st"), Header.PixelSize.X, true, null);
using (TextWriter Writer = File.CreateText(Path.Combine(Options.Import.Folder, "imod", entry.Name, entry.Name + ".rawtlt")))
{
foreach (var angle in entry.TiltAngles)
{
Writer.WriteLine(angle.ToString(CultureInfo.InvariantCulture));
}
}
//HeaderMRC H = (HeaderMRC)MapHeader.ReadFromFile(Path.Combine(Options.Import.Folder, "imod", entry.Name, entry.Name + ".st"));
Dispatcher.Invoke(() => ProgressDialog.SetProgress(++idone / (double)_ParsedEntries.Count));
}
});
await ProgressDialog.CloseAsync();
Close?.Invoke();
}
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int layer = -1, Stream stream = null)
{
MapHeader Header = null;
Type ValueType = null;
float[][] Data;
if (MapHeader.GetHeaderType(path) != typeof(HeaderTiff))
{
using (BinaryReader Reader = isBigEndian ? new BinaryReaderBE(OpenWithBigBuffer(path)) : new BinaryReader(OpenWithBigBuffer(path)))
{
Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType);
ValueType = Header.GetValueType();
Data = new float[layer < 0 ? Header.Dimensions.Z : 1][];
for (int z = 0; z < Data.Length; z++)
{
if (layer >= 0)
{
Reader.BaseStream.Seek((long)Header.Dimensions.ElementsSlice() * (long)ImageFormatsHelper.SizeOf(ValueType) * layer, SeekOrigin.Current);
}
byte[] Bytes = Reader.ReadBytes((int)Header.Dimensions.ElementsSlice() * (int)ImageFormatsHelper.SizeOf(ValueType));
Data[z] = new float[(int)Header.Dimensions.ElementsSlice()];
if (isBigEndian)
{
if (ValueType == typeof(short) || ValueType == typeof(ushort))
{
for (int i = 0; i < Bytes.Length / 2; i++)
{
Array.Reverse(Bytes, i * 2, 2);
}
}
else if (ValueType == typeof(int) || ValueType == typeof(float))
{
for (int i = 0; i < Bytes.Length / 4; i++)
{
Array.Reverse(Bytes, i * 4, 4);
}
}
else if (ValueType == typeof(double))
{
for (int i = 0; i < Bytes.Length / 8; i++)
{
Array.Reverse(Bytes, i * 8, 8);
}
}
}
unsafe
{
int Elements = (int)Header.Dimensions.ElementsSlice();
fixed(byte *BytesPtr = Bytes)
fixed(float *DataPtr = Data[z])
{
float *DataP = DataPtr;
if (ValueType == typeof(byte))
{
byte *BytesP = BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(short))
{
short *BytesP = (short *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(ushort))
{
ushort *BytesP = (ushort *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(int))
{
int *BytesP = (int *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(float))
{
float *BytesP = (float *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = *BytesP++;
}
}
else if (ValueType == typeof(double))
{
double *BytesP = (double *)BytesPtr;
for (int i = 0; i < Elements; i++)
{
*DataP++ = (float)*BytesP++;
}
}
}
}
}
}
}
else
{
Header = MapHeader.ReadFromFile(null, path, headerlessSliceDims, headerlessOffset, headerlessType, stream);
Data = ((HeaderTiff)Header).ReadData(stream, layer);
}
return(Data);
}
private async void ButtonWrite_OnClick(object sender, RoutedEventArgs e)
{
System.Windows.Forms.SaveFileDialog FileDialog = new System.Windows.Forms.SaveFileDialog
{
Filter = "STAR Files|*.star"
};
System.Windows.Forms.DialogResult Result = FileDialog.ShowDialog();
if (Result.ToString() == "OK")
{
ExportPath = FileDialog.FileName;
}
else
{
return;
}
bool Relative = (bool)CheckRelative.IsChecked;
bool Filter = (bool)CheckFilter.IsChecked;
bool Manual = (bool)CheckManual.IsChecked;
ProgressWrite.Visibility = Visibility.Visible;
PanelButtons.Visibility = Visibility.Hidden;
await Task.Run(() =>
{
List <Movie> ValidMovies = Movies.Where(v =>
{
if (!Filter && v.UnselectFilter && v.UnselectManual == null)
{
return(false);
}
if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual)
{
return(false);
}
if (Options.Tasks.MicListMakePolishing && (v.OptionsMovement == null || v.GridMovementX.Dimensions.Z <= 1))
{
return(false);
}
return(true);
}).ToList();
string PathPrefix = "", PathPrefixAverage = "", GainPath = "";
if (ValidMovies.Count > 0 && Relative)
{
Uri UriStar = new Uri(ExportPath);
PathPrefix = Helper.PathToFolder(UriStar.MakeRelativeUri(new Uri(ValidMovies[0].Path)).ToString());
if (PathPrefix == ValidMovies[0].Name)
{
PathPrefix = "";
}
PathPrefixAverage = Helper.PathToFolder(UriStar.MakeRelativeUri(new Uri(ValidMovies[0].AveragePath)).ToString());
if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain)
{
// Since RELION can't read DM4 or flip/transpose the gain, save a transformed version as MRC.
string TransformedGainPath = Helper.PathToFolder(Options.Import.GainPath) + "gain/transformed.mrc";
Directory.CreateDirectory(Helper.PathToFolder(TransformedGainPath));
Image TransformedGain = MainWindow.LoadAndPrepareGainReference();
TransformedGain.WriteMRC(TransformedGainPath, (float)Options.PixelSizeMean, true);
TransformedGain.Dispose();
GainPath = UriStar.MakeRelativeUri(new Uri(TransformedGainPath)).ToString();
}
}
Dispatcher.Invoke(() => ProgressWrite.Maximum = ValidMovies.Count);
bool IncludeCTF = ValidMovies.Any(v => v.OptionsCTF != null);
bool IncludePolishing = ValidMovies.Any(v => v.OptionsMovement != null) && Options.Tasks.MicListMakePolishing;
Star TableOut = new Star(new[] { "rlnMicrographName" });
if (IncludeCTF)
{
TableOut.AddColumn("rlnMagnification");
TableOut.AddColumn("rlnDetectorPixelSize");
TableOut.AddColumn("rlnVoltage");
TableOut.AddColumn("rlnSphericalAberration");
TableOut.AddColumn("rlnAmplitudeContrast");
TableOut.AddColumn("rlnPhaseShift");
TableOut.AddColumn("rlnDefocusU");
TableOut.AddColumn("rlnDefocusV");
TableOut.AddColumn("rlnDefocusAngle");
TableOut.AddColumn("rlnCtfMaxResolution");
TableOut.AddColumn("rlnBeamTiltX");
TableOut.AddColumn("rlnBeamTiltY");
}
if (IncludePolishing)
{
TableOut.AddColumn("rlnMicrographMetadata");
}
List <Action> TablesToWrite = new List <Action>();
if (IncludePolishing && ValidMovies.Count > 0)
{
Directory.CreateDirectory(ValidMovies[0].DirectoryName + "motion");
}
int r = 0;
Parallel.ForEach(ValidMovies, new ParallelOptions()
{
MaxDegreeOfParallelism = 4
}, movie =>
{
List <string> Row = new List <string>()
{
PathPrefixAverage + movie.RootName + ".mrc"
};
if (movie.OptionsCTF != null)
{
Row.AddRange(new[]
{
(1e4M / movie.OptionsCTF.BinnedPixelSizeMean).ToString("F1", CultureInfo.InvariantCulture),
"1.0",
movie.CTF.Voltage.ToString("F1", CultureInfo.InvariantCulture),
movie.CTF.Cs.ToString("F4", CultureInfo.InvariantCulture),
movie.CTF.Amplitude.ToString("F3", CultureInfo.InvariantCulture),
(movie.CTF.PhaseShift * 180M).ToString("F1", CultureInfo.InvariantCulture),
((movie.CTF.Defocus + movie.CTF.DefocusDelta / 2) * 1e4M).ToString("F1", CultureInfo.InvariantCulture),
((movie.CTF.Defocus - movie.CTF.DefocusDelta / 2) * 1e4M).ToString("F1", CultureInfo.InvariantCulture),
movie.CTF.DefocusAngle.ToString("F1", CultureInfo.InvariantCulture),
movie.CTFResolutionEstimate.ToString("F1", CultureInfo.InvariantCulture),
movie.CTF.BeamTilt.X.ToString("F2", CultureInfo.InvariantCulture),
movie.CTF.BeamTilt.Y.ToString("F2", CultureInfo.InvariantCulture)
});
}
else
{
Row.AddRange(Helper.ArrayOfFunction(i => "0.0", 12));
}
if (IncludePolishing)
{
Row.Add(PathPrefix + "motion/" + movie.RootName + ".star");
}
lock (TableOut)
TableOut.AddRow(Row);
if (IncludePolishing)
{
MapHeader Header = MapHeader.ReadFromFile(movie.Path);
List <string> HeaderNames = new List <string>()
{
"rlnImageSizeX",
"rlnImageSizeY",
"rlnImageSizeZ",
"rlnMicrographMovieName",
"rlnMicrographBinning",
"rlnMicrographOriginalPixelSize",
"rlnMicrographDoseRate",
"rlnMicrographPreExposure",
"rlnVoltage",
"rlnMicrographStartFrame",
"rlnMotionModelVersion"
};
List <string> HeaderValues = new List <string>()
{
Header.Dimensions.X.ToString(),
Header.Dimensions.Y.ToString(),
Header.Dimensions.Z.ToString(),
PathPrefix + movie.Name,
Math.Pow(2.0, (double)Options.Import.BinTimes).ToString("F5"),
Options.PixelSizeMean.ToString("F5"),
Options.Import.DosePerAngstromFrame.ToString("F5"),
"0",
Options.CTF.Voltage.ToString(),
"1",
"0"
};
if (!string.IsNullOrEmpty(GainPath))
{
HeaderNames.Add("rlnMicrographGainName");
HeaderValues.Add(GainPath);
}
StarParameters ParamsTable = new StarParameters(HeaderNames.ToArray(), HeaderValues.ToArray());
float2[] MotionTrack = movie.GetMotionTrack(new float2(0.5f, 0.5f), 1).Select(v => v / (float)Options.PixelSizeMean).ToArray();
Star TrackTable = new Star(new[]
{
Helper.ArrayOfSequence(1, MotionTrack.Length + 1, 1).Select(v => v.ToString()).ToArray(),
MotionTrack.Select(v => (-v.X).ToString("F5")).ToArray(),
MotionTrack.Select(v => (-v.Y).ToString("F5")).ToArray()
},
new[]
{
"rlnMicrographFrameNumber",
"rlnMicrographShiftX",
"rlnMicrographShiftY"
});
Star.SaveMultitable(movie.DirectoryName + "motion/" + movie.RootName + ".star", new Dictionary <string, Star>()
{
{ "general", ParamsTable },
{ "global_shift", TrackTable },
});
}
lock (this)
{
r++;
if (r % 10 == 0 || r == ValidMovies.Count)
{
Dispatcher.Invoke(() => ProgressWrite.Value = r);
}
}
});
TableOut.Save(ExportPath);
});
Close?.Invoke();
}
public void UpdateDepiction()
{
if (DepictionMesh != null)
{
DepictionMesh.Dispose();
DepictionMesh = null;
}
foreach (var point in Points)
{
point.DropDepictionMesh();
}
//if (Points.Count == 0)
//return;
if (Depiction == PointDepiction.Mesh && File.Exists(DepictionMeshPath))
{
FileInfo Info = new FileInfo(DepictionMeshPath);
if (Info.Extension.ToLower().Contains("mrc"))
{
HeaderMRC VolumeHeader = (HeaderMRC)MapHeader.ReadFromFile(DepictionMeshPath);
float[] VolumeData = IOHelper.ReadSmallMapFloat(DepictionMeshPath, new int2(1, 1), 0, typeof(float));
Mesh NewMesh = Mesh.FromVolume(VolumeData, VolumeHeader.Dimensions, VolumeHeader.Pixelsize.X, (float)DepictionMeshLevel);
NewMesh.UsedComponents = MeshVertexComponents.Position | MeshVertexComponents.Normal;
NewMesh.GLContext = MainWindow.Options.Viewport.GetControl();
NewMesh.UpdateBuffers();
_DepictionMesh = NewMesh;
}
else if (Info.Extension.ToLower().Contains("obj"))
{
Mesh NewMesh = Mesh.FromOBJ(DepictionMeshPath, true);
NewMesh.UsedComponents = MeshVertexComponents.Position | MeshVertexComponents.Normal;
NewMesh.GLContext = MainWindow.Options.Viewport.GetControl();
NewMesh.UpdateBuffers();
_DepictionMesh = NewMesh;
}
}
else if (Depiction == PointDepiction.LocalSurface && MainWindow.Options.Membrane.TomogramTexture != null)
{
int3 DimsExtract = new int3((int)Size + 2, (int)Size + 2, (int)Size + 2);
Parallel.ForEach(Points, point =>
{
Vector3 TomoPos = (point.Position - MainWindow.Options.Membrane.TomogramTexture.Offset) / MainWindow.Options.PixelScale.X;
int3 TomoPosInt = new int3((int)Math.Round(TomoPos.X), (int)Math.Round(TomoPos.Y), (int)Math.Round(TomoPos.Z));
float[] LocalVol = Helper.Extract(MainWindow.Options.Membrane.TomogramTexture.OriginalData,
MainWindow.Options.Membrane.TomogramTexture.Size,
TomoPosInt,
DimsExtract);
if (DepictionLocalSurfaceInvert)
{
for (int i = 0; i < LocalVol.Length; i++)
{
LocalVol[i] = -LocalVol[i];
}
}
for (int z = 0; z < DimsExtract.Z; z++)
{
for (int y = 0; y < DimsExtract.Y; y++)
{
for (int x = 0; x < DimsExtract.X; x++)
{
if (z == 0 || y == 0 || x == 0 || z == DimsExtract.Z - 1 || y == DimsExtract.Y - 1 || x == DimsExtract.X - 1)
{
LocalVol[(z * DimsExtract.Y + y) * DimsExtract.X + x] = -99999;
}
}
}
}
bool[] Mask = new bool[LocalVol.Length];
float Threshold = (float)DepictionLocalSurfaceLevel;
if (DepictionLocalSurfaceOnlyCenter)
{
int MostCentralID = -1;
float MostCentralDist = DimsExtract.X * DimsExtract.X;
// Find most central valid pixel in the local window to start mask expansion from there.
for (int z = 1; z < DimsExtract.Z - 1; z++)
{
int zz = z - DimsExtract.Z / 2;
zz *= zz;
for (int y = 1; y < DimsExtract.Y - 1; y++)
{
int yy = y - DimsExtract.Y / 2;
yy *= yy;
for (int x = 1; x < DimsExtract.X - 1; x++)
{
if (LocalVol[(z * DimsExtract.Y + y) * DimsExtract.X + x] >= Threshold)
{
int xx = x - DimsExtract.X / 2;
xx *= xx;
float r = xx + yy + zz;
if (r < MostCentralDist)
{
MostCentralDist = r;
MostCentralID = (z * DimsExtract.Y + y) * DimsExtract.X + x;
}
}
}
}
}
if (MostCentralID < 0) // Volume doesn't contain voxels above threshold
{
return;
}
Mask[MostCentralID] = true;
for (int mi = 0; mi < Size / 2; mi++)
{
bool[] NextMask = new bool[Mask.Length];
for (int z = 1; z < DimsExtract.Z - 1; z++)
{
for (int y = 1; y < DimsExtract.Y - 1; y++)
{
for (int x = 1; x < DimsExtract.X - 1; x++)
{
int ID = (z * DimsExtract.Y + y) * DimsExtract.X + x;
if (LocalVol[ID] >= Threshold)
{
if (Mask[ID] ||
Mask[ID + 1] ||
Mask[ID - 1] ||
Mask[ID + DimsExtract.X] ||
Mask[ID - DimsExtract.X] ||
Mask[ID + DimsExtract.Y * DimsExtract.X] ||
Mask[ID - DimsExtract.Y * DimsExtract.X])
{
NextMask[ID] = true;
}
}
}
}
}
Mask = NextMask;
}
}
else
{
for (int i = 0; i < Mask.Length; i++)
{
Mask[i] = true;
}
}
// Apply spherical mask
int Size2 = (int)(Size * Size / 4);
for (int z = 1; z < DimsExtract.Z - 1; z++)
{
int zz = z - DimsExtract.Z / 2;
zz *= zz;
for (int y = 1; y < DimsExtract.Y - 1; y++)
{
int yy = y - DimsExtract.Y / 2;
yy *= yy;
for (int x = 1; x < DimsExtract.X - 1; x++)
{
int xx = x - DimsExtract.X / 2;
xx *= xx;
int r2 = xx + yy + zz;
Mask[(z * DimsExtract.Y + y) * DimsExtract.X + x] &= r2 < Size2;
}
}
}
for (int i = 0; i < Mask.Length; i++)
{
if (!Mask[i])
{
LocalVol[i] = Math.Min(LocalVol[i], Threshold - 1e-5f);
}
}
//IOHelper.WriteMapFloat("d_extract.mrc", HeaderMRC.ReadFromFile("test_extract.mrc"), LocalVol);
//IOHelper.WriteMapFloat("d_original.mrc", HeaderMRC.ReadFromFile("Tomo1L1_bin4.mrc"), MainWindow.Options.Membrane.TomogramTexture.OriginalData);
point.DepictionMesh = Mesh.FromVolume(LocalVol,
DimsExtract,
MainWindow.Options.PixelScale.X,
(float)DepictionLocalSurfaceLevel);
point.DepictionMesh.UsedComponents = MeshVertexComponents.Position | MeshVertexComponents.Normal;
});
foreach (var point in Points)
{
if (point.DepictionMesh == null)
{
continue;
}
point.DepictionMesh.GLContext = MainWindow.Options.Viewport.GetControl();
point.DepictionMesh.UpdateBuffers();
}
}
MainWindow.Options.Viewport.Redraw();
}
static void Main(string[] args)
{
Console.WriteLine("Path to stacks:");
string FolderPath = Console.ReadLine();
if (FolderPath[FolderPath.Length - 1] != '\\' || FolderPath[FolderPath.Length - 1] != '/')
{
FolderPath += "\\";
}
Console.WriteLine("Reading stacks from " + FolderPath + "\n");
Console.WriteLine("Look for files recursively in subfolders? (y/n)");
bool DoRecursiveSearch = Console.ReadLine().ToLower().Contains("y");
Console.WriteLine($"Files will {(DoRecursiveSearch ? "" : "not ")}be searched recursively.\n");
Console.WriteLine("Output folder (leave empty if identical with input):");
string OutputPath = Console.ReadLine();
if (string.IsNullOrEmpty(OutputPath))
{
OutputPath = FolderPath;
}
if (OutputPath[OutputPath.Length - 1] != '\\' || OutputPath[OutputPath.Length - 1] != '/')
{
OutputPath += "\\";
}
Console.WriteLine("Writing compressed stacks to " + OutputPath + "\n");
Console.WriteLine("Compress to TIFF (c), or just move (m)? (c/m)");
bool Compress = Console.ReadLine().ToLower().Contains("c");
Console.WriteLine($"Files will be {(Compress ? "written as compressed TIFFs" : "just moved")}.\n");
string Extension = "mrc";
if (!Compress)
{
Console.WriteLine("What is the input file extension?");
Extension = Console.ReadLine().ToLower();
if (Extension[0] == '*')
{
Extension = Extension.Substring(1);
}
if (Extension[0] == '.')
{
Extension = Extension.Substring(1);
}
Console.WriteLine($"Using *.{Extension} as input file extension.\n");
}
Console.WriteLine("Number of frames:");
string FramesString = Console.ReadLine();
int NFrames;
try
{
NFrames = int.Parse(FramesString);
Console.WriteLine("Using " + NFrames + " frames.\n");
}
catch (Exception)
{
return;
}
Console.WriteLine("Delete original files after completion? (y/n)");
bool DeleteWhenDone = Console.ReadLine().ToLower().Contains("y");
Console.WriteLine($"Original files will {(DeleteWhenDone? "" : "not ")}be deleted.\n");
if (!DeleteWhenDone)
{
Directory.CreateDirectory(FolderPath + "original");
}
Console.WriteLine("Delete superfluous gain references? (y/n)");
bool DeleteExtraGain = Console.ReadLine().ToLower().Contains("y");
Console.WriteLine($"Superfluous gain references will {(DeleteWhenDone ? "" : "not ")}be deleted.\n");
if (!DeleteWhenDone)
{
Directory.CreateDirectory(FolderPath + "original");
}
Console.WriteLine("Number of stacks to be processed in parallel (4 is a good value):");
int NParallel = int.Parse(Console.ReadLine());
Console.WriteLine($"{NParallel} stacks will be processed in parallel.\n");
GC.AddMemoryPressure(20 * ((long)1 << 30));
List <string> HaveBeenProcessed = new List <string>();
while (true)
{
List <string> FrameNames = new List <string>();
List <string> GainRefNames = new List <string>();
foreach (var filename in Directory.EnumerateFiles(FolderPath, "*." + Extension,
DoRecursiveSearch ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
try
{
MapHeader Header = MapHeader.ReadFromFile(filename);
if (Header.Dimensions.Z != NFrames)
{
if (Header.Dimensions.Z == 1)
{
GainRefNames.Add(filename);
}
continue;
}
if (HaveBeenProcessed.Contains(filename))
{
continue;
}
if (DeleteWhenDone)
{
FrameNames.Add(filename);
}
else if (!filename.Contains("\\original\\"))
{
FrameNames.Add(filename);
}
}
catch
{
}
}
int NFiles = FrameNames.Count;
if (NFiles == 0)
{
Thread.Sleep(1000);
continue;
}
Console.WriteLine("Found " + NFiles + " new stacks.");
Thread.Sleep(1000);
SemaphoreSlim WritingSemaphore = new SemaphoreSlim(NParallel);
for (int f = 0; f < NFiles; f++)
{
string FrameName = FrameNames[f];
MapHeader Header = MapHeader.ReadFromFilePatient(500, 100, FrameName, new int2(1), 0, typeof(float));
if (Compress)
{
Image StackOut = new Image(Header.Dimensions);
float[][] StackOutData = StackOut.GetHost(Intent.Read);
for (int n = 0; n < NFrames; n++)
{
Image Frame = Image.FromFilePatient(50, 100, FrameName, n);
float[] FrameData = Frame.GetHost(Intent.Read)[0];
if (Compress)
{
for (int i = 0; i < FrameData.Length; i++)
{
StackOutData[n][i] = (float)Math.Max(0, Math.Min(255, Math.Round(FrameData[i])));
}
}
else
{
for (int i = 0; i < FrameData.Length; i++)
{
StackOutData[n][i] = FrameData[i];
}
}
Frame.Dispose();
Console.Write(".");
}
Console.WriteLine("");
HaveBeenProcessed.Add(FrameName);
string RootName = Helper.PathToName(FrameName);
Thread WriteThread = new Thread(() =>
{
try
{
//if (Compress)
StackOut.WriteTIFF(OutputPath + RootName + ".tif", 1, typeof(byte));
//else
// StackOut.WriteMRC(OutputPath + RootName + ".mrc", 1, true);
StackOut.Dispose();
if (DeleteWhenDone)
{
File.Delete(FrameName);
}
else
{
File.Move(FrameName, FolderPath + "original/" + Helper.PathToNameWithExtension(FrameName));
}
}
catch (Exception exc)
{
Console.WriteLine("ERROR: Could not write " + RootName);
Console.WriteLine(exc);
HaveBeenProcessed.Remove(FrameName);
}
WritingSemaphore.Release();
GC.Collect();
});
while (WritingSemaphore.CurrentCount < 1)
{
Thread.Sleep(100);
}
WritingSemaphore.Wait();
WriteThread.Start();
Console.WriteLine("Done reading: " + RootName);
}
else
{
bool Success = false;
while (!Success)
{
try
{
string NameOut = OutputPath + Helper.PathToNameWithExtension(FrameName);
if (DeleteWhenDone)
{
File.Move(FrameName, NameOut);
}
else
{
File.Copy(FrameName, NameOut);
File.Move(FrameName, FolderPath + "original/" + Helper.PathToNameWithExtension(FrameName));
}
HaveBeenProcessed.Add(FrameName);
Success = true;
Console.WriteLine("Done moving: " + Helper.PathToNameWithExtension(FrameName));
}
catch (Exception exc)
{
Console.WriteLine("Something went wrong moving " + Helper.PathToNameWithExtension(FrameName) + ":\n" + exc.ToString());
}
}
}
}
if (DeleteExtraGain)
{
foreach (var gainRefName in GainRefNames)
{
File.Delete(gainRefName);
}
}
Thread.Sleep(1000);
}
}
public static float[][] ReadMapFloat(BinaryReader reader, string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int[] layers = null, Stream stream = null, float[][] reuseBuffer = null)
{
MapHeader Header = null;
Type ValueType = null;
float[][] Data;
BinaryReader Reader = reader;
{
Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType);
ValueType = Header.GetValueType();
Data = reuseBuffer == null ? new float[layers == null ? Header.Dimensions.Z : layers.Length][] : reuseBuffer;
int ReadBatchSize = Math.Min((int)Header.Dimensions.ElementsSlice(), 1 << 20);
int ValueSize = (int)ImageFormatsHelper.SizeOf(ValueType);
byte[] Bytes = new byte[ReadBatchSize * ValueSize];
long ReaderDataStart = Reader.BaseStream.Position;
for (int z = 0; z < Data.Length; z++)
{
if (layers != null)
{
Reader.BaseStream.Seek(Header.Dimensions.ElementsSlice() * ImageFormatsHelper.SizeOf(ValueType) * layers[z] + ReaderDataStart, SeekOrigin.Begin);
}
if (reuseBuffer == null)
{
Data[z] = new float[(int)Header.Dimensions.ElementsSlice()];
}
unsafe
{
fixed(byte *BytesPtr = Bytes)
fixed(float *DataPtr = Data[z])
{
for (int b = 0; b < (int)Header.Dimensions.ElementsSlice(); b += ReadBatchSize)
{
int CurBatch = Math.Min(ReadBatchSize, (int)Header.Dimensions.ElementsSlice() - b);
Reader.Read(Bytes, 0, CurBatch * ValueSize);
if (isBigEndian)
{
if (ValueType == typeof(short) || ValueType == typeof(ushort))
{
for (int i = 0; i < CurBatch * ValueSize / 2; i++)
{
Array.Reverse(Bytes, i * 2, 2);
}
}
else if (ValueType == typeof(int) || ValueType == typeof(float))
{
for (int i = 0; i < CurBatch * ValueSize / 4; i++)
{
Array.Reverse(Bytes, i * 4, 4);
}
}
else if (ValueType == typeof(double))
{
for (int i = 0; i < CurBatch * ValueSize / 8; i++)
{
Array.Reverse(Bytes, i * 8, 8);
}
}
}
float *DataP = DataPtr + b;
if (ValueType == typeof(byte))
{
byte *BytesP = BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(short))
{
short *BytesP = (short *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(ushort))
{
ushort *BytesP = (ushort *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(int))
{
int *BytesP = (int *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(float))
{
float *BytesP = (float *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = *BytesP++;
}
}
else if (ValueType == typeof(double))
{
double *BytesP = (double *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
}
}
}
}
}
return(Data);
}
private async void Movie_PS2DChanged(object sender, EventArgs e)
{
try
{
AdjustGridVisibility();
ImagePS2D.Visibility = Visibility.Hidden;
if (Movie == null)
{
return;
}
if (Movie.OptionsCTF == null || !File.Exists(Movie.PowerSpectrumPath))
{
return;
}
// Check if 2D PS file has the PS for the currently selected tilt
if (Movie.GetType() == typeof(TiltSeries))
{
MapHeader Header = MapHeader.ReadFromFile(Movie.PowerSpectrumPath);
if (Header.Dimensions.Z <= TiltID)
{
return;
}
}
ProgressPS2D.Visibility = Visibility.Visible;
Movie movie = Movie;
//await Task.Delay(1000);
await Task.Run(() =>
{
ImageSource PS2D;
unsafe
{
int Slice = -1;
if (movie.GetType() == typeof(TiltSeries))
{
Slice = TiltID;
}
MapHeader Header = MapHeader.ReadFromFile(movie.PowerSpectrumPath);
float[] Data = Image.FromFile(movie.PowerSpectrumPath, Slice).GetHostContinuousCopy();
int Width = Header.Dimensions.X;
int HalfWidth = Width / 2;
int Height = Header.Dimensions.Y; // The usual x / 2 + 1
int RadiusMin2 = (int)(movie.OptionsCTF.RangeMin *HalfWidth);
RadiusMin2 *= RadiusMin2;
int RadiusMax2 = (int)(movie.OptionsCTF.RangeMax *HalfWidth);
RadiusMax2 *= RadiusMax2;
double Sum1 = 0;
double Sum2 = 0;
int Samples = 0;
fixed(float *DataPtr = Data)
{
float *DataP = DataPtr;
for (int y = 0; y < Height; y++)
{
for (int x = 0; x < Width; x++)
{
int XCentered = x - HalfWidth;
int YCentered = Height - 1 - y;
int Radius2 = XCentered * XCentered + YCentered * YCentered;
if (Radius2 >= RadiusMin2 && Radius2 <= RadiusMax2)
{
Sum1 += *DataP;
Sum2 += (*DataP) * (*DataP);
Samples++;
}
DataP++;
}
}
float Mean = (float)(Sum1 / Samples);
float Std = (float)(Math.Sqrt(Samples *Sum2 - Sum1 *Sum1) / Samples);
float ValueMin = Mean - 1.5f * Std;
float ValueMax = Mean + 3.0f * Std;
float Range = ValueMax - ValueMin;
if (Range <= 0f)
{
PS2D = BitmapSource.Create(Width, Height, 96, 96, PixelFormats.Indexed8, BitmapPalettes.Gray256, new byte[Data.Length], Width);
}
else
{
byte[] DataBytes = new byte[Data.Length];
fixed(byte *DataBytesPtr = DataBytes)
{
for (int y = 0; y < Height; y++)
{
for (int x = 0; x < Width; x++)
{
DataBytesPtr[(Height - 1 - y) * Width + x] = (byte)(Math.Max(Math.Min(1f, (DataPtr[y * Width + x] - ValueMin) / Range), 0f) * 255f);
}
}
}
PS2D = BitmapSource.Create(Width, Height, 96, 96, PixelFormats.Indexed8, BitmapPalettes.Gray256, DataBytes, Width);
PS2D.Freeze();
}
Dispatcher.Invoke(() =>
{
ImagePS2D.Source = PS2D;
ImagePS2D.Visibility = Visibility.Visible;
});
}
}
});
ProgressPS2D.Visibility = Visibility.Hidden;
}
catch { }
}
private async void ButtonMatch_OnClick(object sender, RoutedEventArgs e)
{
bool Filter = (bool)CheckFilter.IsChecked;
bool Manual = (bool)CheckManual.IsChecked;
#region Get all movies that can potentially be used
List <Movie> ValidMovies = Movies.Where(v =>
{
if (!Filter && v.UnselectFilter && v.UnselectManual == null)
{
return(false);
}
if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual)
{
return(false);
}
if (v.OptionsCTF == null)
{
return(false);
}
return(true);
}).ToList();
if (ValidMovies.Count == 0)
{
Close?.Invoke();
return;
}
#endregion
#region Set up progress displays
NParallel = Math.Min(ValidMovies.Count, GPU.GetDeviceCount());
GridSizes = Helper.ArrayOfConstant(new int3(1), NParallel);
GridProgress = new int[NParallel];
GridNames = new string[NParallel];
GridControls = Helper.ArrayOfFunction(i => new FlatGridMiniature()
{
Width = 256
}, NParallel);
GridLabels = Helper.ArrayOfFunction(i => new TextBlock {
FontSize = 18, Margin = new Thickness(0, 15, 0, 0), HorizontalAlignment = HorizontalAlignment.Center
}, NParallel);
for (int n = 0; n < NParallel; n++)
{
StackPanel SP = new StackPanel {
Orientation = Orientation.Vertical, Margin = new Thickness(15, 0, 15, 0)
};
SP.Children.Add(GridControls[n]);
SP.Children.Add(GridLabels[n]);
PanelGrids.Children.Add(SP);
}
#endregion
PanelConfiguration.Visibility = Visibility.Collapsed;
PanelProgress.Visibility = Visibility.Visible;
int Completed = 0;
ProgressOverall.Maximum = ValidMovies.Count;
await Task.Run(() =>
{
#region Load template and make copies for all devices
Image[] Template = new Image[NParallel];
{
Template[0] = Image.FromFile(PathTemplate);
Template[0].PixelSize = (float)Options.Tasks.TomoMatchTemplatePixel;
for (int i = 0; i < Template.Length; i++)
{
Template[i] = Template[0].GetCopy();
}
}
#endregion
#region Load gain reference if needed
Image[] ImageGain = new Image[NParallel];
DefectModel[] DefectMap = new DefectModel[NParallel];
if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain && File.Exists(Options.Import.GainPath))
{
for (int d = 0; d < NParallel; d++)
{
ImageGain[d] = MainWindow.LoadAndPrepareGainReference();
}
}
if (!string.IsNullOrEmpty(Options.Import.DefectsPath) && Options.Import.CorrectDefects && File.Exists(Options.Import.DefectsPath))
{
for (int d = 0; d < NParallel; d++)
{
DefectMap[d] = MainWindow.LoadAndPrepareDefectMap();
}
}
#endregion
object SyncDummy = new object();
Helper.ForEachGPU(ValidMovies, (item, gpuID) =>
{
if (IsCanceled)
{
return(true); // This cancels the iterator
}
Dispatcher.Invoke(() =>
{
SetGridSize(gpuID, new int3(1, 1, 1));
SetGridProgress(gpuID, 0);
SetGridName(gpuID, "Loading data...");
});
ProcessingOptionsFullMatch MatchOptions = Options.GetProcessingFullMatch();
MatchOptions.TemplateName = Helper.PathToName(PathTemplate);
#region Load and prepare original movie
Image OriginalStack = null;
decimal ScaleFactor = 1M / MatchOptions.DownsampleFactor;
MapHeader OriginalHeader = MapHeader.ReadFromFile(item.Path);
MainWindow.LoadAndPrepareHeaderAndMap(item.Path, ImageGain[gpuID], DefectMap[gpuID], ScaleFactor, out OriginalHeader, out OriginalStack);
if (IsCanceled)
{
OriginalStack?.Dispose();
return(true);
}
#endregion
item.MatchFull(OriginalStack, MatchOptions, Template[gpuID], (size, value, name) =>
{
Dispatcher.Invoke(() =>
{
SetGridSize(gpuID, size);
SetGridProgress(gpuID, value);
SetGridName(gpuID, name);
});
return(IsCanceled);
});
OriginalStack?.Dispose();
Dispatcher.Invoke(() => ProgressOverall.Value = ++Completed);
return(false); // No need to cancel GPU ForEach iterator
}, 1);
});
Close?.Invoke();
}
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int[] layers = null, Stream stream = null, float[][] reuseBuffer = null)
{
MapHeader Header = null;
Type ValueType = null;
float[][] Data;
if (MapHeader.GetHeaderType(path) != typeof(HeaderTiff))
{
using (BinaryReader Reader = isBigEndian ? new BinaryReaderBE(OpenWithBigBuffer(path)) : new BinaryReader(OpenWithBigBuffer(path)))
{
Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType);
ValueType = Header.GetValueType();
Data = reuseBuffer == null ? new float[layers == null ? Header.Dimensions.Z : layers.Length][] : reuseBuffer;
int ReadBatchSize = Math.Min((int)Header.Dimensions.ElementsSlice(), 1 << 20);
int ValueSize = (int)ImageFormatsHelper.SizeOf(ValueType);
byte[] Bytes = new byte[ReadBatchSize * ValueSize];
long ReaderDataStart = Reader.BaseStream.Position;
for (int z = 0; z < Data.Length; z++)
{
if (layers != null)
{
Reader.BaseStream.Seek(Header.Dimensions.ElementsSlice() * ImageFormatsHelper.SizeOf(ValueType) * layers[z] + ReaderDataStart, SeekOrigin.Begin);
}
if (reuseBuffer == null)
{
Data[z] = new float[(int)Header.Dimensions.ElementsSlice()];
}
unsafe
{
fixed(byte *BytesPtr = Bytes)
fixed(float *DataPtr = Data[z])
{
for (int b = 0; b < (int)Header.Dimensions.ElementsSlice(); b += ReadBatchSize)
{
int CurBatch = Math.Min(ReadBatchSize, (int)Header.Dimensions.ElementsSlice() - b);
Reader.Read(Bytes, 0, CurBatch * ValueSize);
if (isBigEndian)
{
if (ValueType == typeof(short) || ValueType == typeof(ushort))
{
for (int i = 0; i < CurBatch * ValueSize / 2; i++)
{
Array.Reverse(Bytes, i * 2, 2);
}
}
else if (ValueType == typeof(int) || ValueType == typeof(float))
{
for (int i = 0; i < CurBatch * ValueSize / 4; i++)
{
Array.Reverse(Bytes, i * 4, 4);
}
}
else if (ValueType == typeof(double))
{
for (int i = 0; i < CurBatch * ValueSize / 8; i++)
{
Array.Reverse(Bytes, i * 8, 8);
}
}
}
float *DataP = DataPtr + b;
if (ValueType == typeof(byte))
{
byte *BytesP = BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(short))
{
short *BytesP = (short *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(ushort))
{
ushort *BytesP = (ushort *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(int))
{
int *BytesP = (int *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
else if (ValueType == typeof(float))
{
float *BytesP = (float *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = *BytesP++;
}
}
else if (ValueType == typeof(double))
{
double *BytesP = (double *)BytesPtr;
for (int i = 0; i < CurBatch; i++)
{
*DataP++ = (float)*BytesP++;
}
}
}
}
}
}
}
}
else
{
Header = MapHeader.ReadFromFile(null, path, headerlessSliceDims, headerlessOffset, headerlessType, stream);
if (Helper.PathToExtension(path).ToLower() == ".eer")
{
Data = Helper.ArrayOfFunction(i => new float[Header.Dimensions.ElementsSlice()], layers == null ? Header.Dimensions.Z : layers.Length);
for (int i = 0; i < Data.Length; i++)
{
int z = layers == null ? i : layers[i];
EERNative.ReadEER(path, z * HeaderEER.GroupNFrames, (z + 1) * HeaderEER.GroupNFrames, HeaderEER.SuperResolution, Data[i]);
}
}
else
{
Data = ((HeaderTiff)Header).ReadData(stream, layers);
}
}
return(Data);
}
