public BoxNetTrain(string modelName, Options options) { InitializeComponent(); ModelName = modelName; Options = options; TextHeader.Text = "Retrain " + ModelName; TextNewName.Text = ModelName + "_2"; int CorpusSize = 0; if (Directory.Exists(System.IO.Path.Combine(Environment.CurrentDirectory, "boxnet2training"))) { foreach (var fileName in Directory.EnumerateFiles(System.IO.Path.Combine(Environment.CurrentDirectory, "boxnet2training"), "*.tif")) { MapHeader Header = MapHeader.ReadFromFile(fileName); CorpusSize += Header.Dimensions.Z / 3; } } CheckCorpus.Content = $" Also use {CorpusSize} micrographs in\n {System.IO.Path.Combine(Environment.CurrentDirectory, "boxnettraining")}"; if (CorpusSize == 0) { CheckCorpus.IsChecked = false; CheckCorpus.IsEnabled = false; } }
public static MapHeader LoadHeader(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType) { lock (Sync) { return(MapHeader.ReadFromFile(path, headerlessSliceDims, headerlessOffset, headerlessType)); } }
public DialogTomoMatch(TiltSeries[] series, string pathTemplate, Options options) { InitializeComponent(); Series = series; PathTemplate = pathTemplate; Options = options; DataContext = Options; TextTemplateName.Value = Helper.PathToNameWithExtension(pathTemplate); MapHeader Header = MapHeader.ReadFromFile(pathTemplate); if (Math.Abs(Header.PixelSize.X - 1f) > 1e-6f) { Options.Tasks.TomoMatchTemplatePixel = (decimal)Header.PixelSize.X; } }
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, int layer = -1) { MapHeader Header = null; Type ValueType = null; FileInfo Info = new FileInfo(path); float[][] Data; using (BinaryReader Reader = new BinaryReader(OpenWithBigBuffer(path))) { Header = MapHeader.ReadFromFile(Reader, Info, headerlessSliceDims, headerlessOffset, headerlessType); ValueType = Header.GetValueType(); Data = new float[layer < 0 ? Header.Dimensions.Z : 1][]; if (Header.GetType() != typeof(HeaderTiff)) { for (int z = 0; z < Data.Length; z++) { if (layer >= 0) { Reader.BaseStream.Seek((int)Header.Dimensions.ElementsSlice() * (int)ImageFormatsHelper.SizeOf(ValueType) * layer, SeekOrigin.Current); } byte[] Bytes = Reader.ReadBytes((int)Header.Dimensions.ElementsSlice() * (int)ImageFormatsHelper.SizeOf(ValueType)); Data[z] = new float[(int)Header.Dimensions.ElementsSlice()]; unsafe { int Elements = (int)Header.Dimensions.ElementsSlice(); fixed(byte *BytesPtr = Bytes) fixed(float *DataPtr = Data[z]) { float *DataP = DataPtr; if (ValueType == typeof(byte)) { byte *BytesP = BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(short)) { short *BytesP = (short *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(ushort)) { ushort *BytesP = (ushort *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(int)) { int *BytesP = (int *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(float)) { float *BytesP = (float *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = *BytesP++; } } else if (ValueType == typeof(double)) { double *BytesP = (double *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } } } } } else { Data = ((HeaderTiff)Header).ReadData(layer); } } return(Data); }
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int layer = -1, Stream stream = null, float[][] reuseBuffer = null) { MapHeader Header = null; Type ValueType = null; float[][] Data; if (MapHeader.GetHeaderType(path) != typeof(HeaderTiff)) { using (BinaryReader Reader = isBigEndian ? new BinaryReaderBE(OpenWithBigBuffer(path)) : new BinaryReader(OpenWithBigBuffer(path))) { Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType); ValueType = Header.GetValueType(); Data = reuseBuffer == null ? new float[layer < 0 ? Header.Dimensions.Z : 1][] : reuseBuffer; int ReadBatchSize = Math.Min((int)Header.Dimensions.ElementsSlice(), 1 << 20); int ValueSize = (int)ImageFormatsHelper.SizeOf(ValueType); byte[] Bytes = new byte[ReadBatchSize * ValueSize]; for (int z = 0; z < Data.Length; z++) { if (layer >= 0) { Reader.BaseStream.Seek(Header.Dimensions.ElementsSlice() * ImageFormatsHelper.SizeOf(ValueType) * layer, SeekOrigin.Current); } if (reuseBuffer == null) { Data[z] = new float[(int)Header.Dimensions.ElementsSlice()]; } unsafe { fixed(byte *BytesPtr = Bytes) fixed(float *DataPtr = Data[z]) { for (int b = 0; b < (int)Header.Dimensions.ElementsSlice(); b += ReadBatchSize) { int CurBatch = Math.Min(ReadBatchSize, (int)Header.Dimensions.ElementsSlice() - b); Reader.Read(Bytes, 0, CurBatch * ValueSize); if (isBigEndian) { if (ValueType == typeof(short) || ValueType == typeof(ushort)) { for (int i = 0; i < CurBatch * ValueSize / 2; i++) { Array.Reverse(Bytes, i * 2, 2); } } else if (ValueType == typeof(int) || ValueType == typeof(float)) { for (int i = 0; i < CurBatch * ValueSize / 4; i++) { Array.Reverse(Bytes, i * 4, 4); } } else if (ValueType == typeof(double)) { for (int i = 0; i < CurBatch * ValueSize / 8; i++) { Array.Reverse(Bytes, i * 8, 8); } } } float *DataP = DataPtr + b; if (ValueType == typeof(byte)) { byte *BytesP = BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(short)) { short *BytesP = (short *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(ushort)) { ushort *BytesP = (ushort *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(int)) { int *BytesP = (int *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(float)) { float *BytesP = (float *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = *BytesP++; } } else if (ValueType == typeof(double)) { double *BytesP = (double *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } } } } } } } else { Header = MapHeader.ReadFromFile(null, path, headerlessSliceDims, headerlessOffset, headerlessType, stream); Data = ((HeaderTiff)Header).ReadData(stream, layer); } return(Data); }
public static float[] ReadSmallMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType) { FileInfo Info = new FileInfo(path); float[] Data; using (BinaryReader Reader = new BinaryReader(OpenWithBigBuffer(path))) { MapHeader Header = MapHeader.ReadFromFile(Reader, Info, headerlessSliceDims, headerlessOffset, headerlessType); Type ValueType = Header.GetValueType(); Data = new float[Header.Dimensions.Elements()]; byte[] Bytes = Reader.ReadBytes((int)Header.Dimensions.Elements() * (int)ImageFormatsHelper.SizeOf(ValueType)); unsafe { int Elements = (int)Header.Dimensions.Elements(); fixed(byte *BytesPtr = Bytes) fixed(float *DataPtr = Data) { float *DataP = DataPtr; if (ValueType == typeof(byte)) { byte *BytesP = BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(short)) { short *BytesP = (short *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(ushort)) { ushort *BytesP = (ushort *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(int)) { int *BytesP = (int *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(float)) { float *BytesP = (float *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = *BytesP++; } } else if (ValueType == typeof(double)) { double *BytesP = (double *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } } } } return(Data); }
private void PrepareData(string savePath) { float Diameter = 1; Dispatcher.Invoke(() => Diameter = (float)SliderDiameter.Value); string ErrorString = ""; //if (string.IsNullOrEmpty(NewName)) // ErrorString += "No name specified.\n"; if (string.IsNullOrEmpty(SuffixPositive)) { ErrorString += "No positive examples selected.\n"; } if (!string.IsNullOrEmpty(ErrorString)) { throw new Exception(ErrorString); } bool IsNegative = Options.Picking.DataStyle != "cryo"; Movie[] Movies = Options.MainWindow.FileDiscoverer.GetImmutableFiles(); List <Movie> ValidMovies = new List <Movie>(); foreach (var movie in Movies) { if (movie.UnselectManual != null && (bool)movie.UnselectManual) { continue; } if (!File.Exists(movie.AveragePath)) { continue; } bool HasExamples = File.Exists(FolderPositive + movie.RootName + SuffixPositive + ".star"); if (!HasExamples) { continue; } ValidMovies.Add(movie); } if (ValidMovies.Count == 0) { throw new Exception("No movie averages could be found to create training examples. Please process the movies first to create the averages."); } List <Image> AllAveragesBN = new List <Image>(); List <Image> AllLabelsBN = new List <Image>(); List <Image> AllCertainBN = new List <Image>(); int MoviesDone = 0; foreach (var movie in ValidMovies) { MapHeader Header = MapHeader.ReadFromFile(movie.AveragePath); float PixelSize = Header.PixelSize.X; #region Load positions, and possibly move on to next movie List <float2> PosPositive = new List <float2>(); List <float2> PosFalse = new List <float2>(); List <float2> PosUncertain = new List <float2>(); if (File.Exists(FolderPositive + movie.RootName + SuffixPositive + ".star")) { PosPositive.AddRange(Star.LoadFloat2(FolderPositive + movie.RootName + SuffixPositive + ".star", "rlnCoordinateX", "rlnCoordinateY").Select(v => v * PixelSize / BoxNet2.PixelSize)); } //if (PosPositive.Count == 0) // continue; if (File.Exists(FolderFalsePositive + movie.RootName + SuffixFalsePositive + ".star")) { PosFalse.AddRange(Star.LoadFloat2(FolderFalsePositive + movie.RootName + SuffixFalsePositive + ".star", "rlnCoordinateX", "rlnCoordinateY").Select(v => v * PixelSize / BoxNet2.PixelSize)); } if (File.Exists(FolderUncertain + movie.RootName + SuffixUncertain + ".star")) { PosUncertain.AddRange(Star.LoadFloat2(FolderUncertain + movie.RootName + SuffixUncertain + ".star", "rlnCoordinateX", "rlnCoordinateY").Select(v => v * PixelSize / BoxNet2.PixelSize)); } #endregion Image Average = Image.FromFile(movie.AveragePath); int2 Dims = new int2(Average.Dims); Image Mask = null; if (File.Exists(movie.MaskPath)) { Mask = Image.FromFile(movie.MaskPath); } float RadiusParticle = Math.Max(1, Diameter / 2 / BoxNet2.PixelSize); float RadiusPeak = Math.Max(1.5f, Diameter / 2 / BoxNet2.PixelSize / 4); float RadiusFalse = Math.Max(1, Diameter / 2 / BoxNet2.PixelSize); float RadiusUncertain = Math.Max(1, Diameter / 2 / BoxNet2.PixelSize); #region Rescale everything and allocate memory int2 DimsBN = new int2(new float2(Dims) * PixelSize / BoxNet2.PixelSize + 0.5f) / 2 * 2; Image AverageBN = Average.AsScaled(DimsBN); Average.Dispose(); if (IsNegative) { AverageBN.Multiply(-1f); } GPU.Normalize(AverageBN.GetDevice(Intent.Read), AverageBN.GetDevice(Intent.Write), (uint)AverageBN.ElementsSliceReal, 1); Image MaskBN = null; if (Mask != null) { MaskBN = Mask.AsScaled(DimsBN); Mask.Dispose(); } Image LabelsBN = new Image(new int3(DimsBN)); Image CertainBN = new Image(new int3(DimsBN)); CertainBN.Fill(1f); #endregion #region Paint all positive and uncertain peaks for (int i = 0; i < 3; i++) { var positions = (new[] { PosPositive, PosFalse, PosUncertain })[i]; float R = (new[] { RadiusPeak, RadiusFalse, RadiusUncertain })[i]; float R2 = R * R; float Label = (new[] { 1, 4, 0 })[i]; float[] ImageData = (new[] { LabelsBN.GetHost(Intent.ReadWrite)[0], CertainBN.GetHost(Intent.ReadWrite)[0], CertainBN.GetHost(Intent.ReadWrite)[0] })[i]; foreach (var pos in positions) { int2 Min = new int2(Math.Max(0, (int)(pos.X - R)), Math.Max(0, (int)(pos.Y - R))); int2 Max = new int2(Math.Min(DimsBN.X - 1, (int)(pos.X + R)), Math.Min(DimsBN.Y - 1, (int)(pos.Y + R))); for (int y = Min.Y; y <= Max.Y; y++) { float yy = y - pos.Y; yy *= yy; for (int x = Min.X; x <= Max.X; x++) { float xx = x - pos.X; xx *= xx; float r2 = xx + yy; if (r2 <= R2) { ImageData[y * DimsBN.X + x] = Label; } } } } } #endregion #region Add junk mask if there is one if (MaskBN != null) { float[] LabelsBNData = LabelsBN.GetHost(Intent.ReadWrite)[0]; float[] MaskBNData = MaskBN.GetHost(Intent.Read)[0]; for (int i = 0; i < LabelsBNData.Length; i++) { if (MaskBNData[i] > 0.5f) { LabelsBNData[i] = 2; } } } #endregion #region Clean up MaskBN?.Dispose(); AllAveragesBN.Add(AverageBN); AverageBN.FreeDevice(); AllLabelsBN.Add(LabelsBN); LabelsBN.FreeDevice(); AllCertainBN.Add(CertainBN); CertainBN.FreeDevice(); #endregion } #region Figure out smallest dimensions that contain everything int2 DimsCommon = new int2(1); foreach (var image in AllAveragesBN) { DimsCommon.X = Math.Max(DimsCommon.X, image.Dims.X); DimsCommon.Y = Math.Max(DimsCommon.Y, image.Dims.Y); } #endregion #region Put everything in one stack and save Image Everything = new Image(new int3(DimsCommon.X, DimsCommon.Y, AllAveragesBN.Count * 3)); float[][] EverythingData = Everything.GetHost(Intent.ReadWrite); for (int i = 0; i < AllAveragesBN.Count; i++) { if (AllAveragesBN[i].Dims == new int3(DimsCommon)) // No padding needed { EverythingData[i * 3 + 0] = AllAveragesBN[i].GetHost(Intent.Read)[0]; EverythingData[i * 3 + 1] = AllLabelsBN[i].GetHost(Intent.Read)[0]; EverythingData[i * 3 + 2] = AllCertainBN[i].GetHost(Intent.Read)[0]; } else // Padding needed { { Image Padded = AllAveragesBN[i].AsPadded(DimsCommon); AllAveragesBN[i].Dispose(); EverythingData[i * 3 + 0] = Padded.GetHost(Intent.Read)[0]; Padded.FreeDevice(); } { Image Padded = AllLabelsBN[i].AsPadded(DimsCommon); AllLabelsBN[i].Dispose(); EverythingData[i * 3 + 1] = Padded.GetHost(Intent.Read)[0]; Padded.FreeDevice(); } { Image Padded = AllCertainBN[i].AsPadded(DimsCommon); AllCertainBN[i].Dispose(); EverythingData[i * 3 + 2] = Padded.GetHost(Intent.Read)[0]; Padded.FreeDevice(); } } } Everything.WriteTIFF(savePath, BoxNet2.PixelSize, typeof(float)); #endregion }
private async void ButtonExport_OnClick(object sender, RoutedEventArgs e) { System.Windows.Forms.SaveFileDialog SaveDialog = new System.Windows.Forms.SaveFileDialog { Filter = "STAR Files|*.star" }; System.Windows.Forms.DialogResult ResultSave = SaveDialog.ShowDialog(); if (ResultSave.ToString() == "OK") { ExportPath = SaveDialog.FileName; } else { return; } bool DoAverage = (bool)RadioAverage.IsChecked; bool DoStack = (bool)RadioStack.IsChecked; bool DoDenoisingPairs = (bool)RadioDenoising.IsChecked; bool DoOnlyStar = (bool)RadioStar.IsChecked; bool Invert = (bool)CheckInvert.IsChecked; bool Normalize = (bool)CheckNormalize.IsChecked; bool Preflip = (bool)CheckPreflip.IsChecked; float AngPix = (float)Options.Tasks.InputPixelSize; bool Relative = (bool)CheckRelative.IsChecked; bool Filter = (bool)CheckFilter.IsChecked; bool Manual = (bool)CheckManual.IsChecked; int BoxSize = (int)Options.Tasks.Export2DBoxSize; int NormDiameter = (int)Options.Tasks.Export2DParticleDiameter; ProgressWrite.Visibility = Visibility.Visible; ProgressWrite.IsIndeterminate = true; PanelButtons.Visibility = Visibility.Collapsed; PanelRemaining.Visibility = Visibility.Visible; foreach (var element in DisableWhileProcessing) { element.IsEnabled = false; } await Task.Run(async() => { #region Get all movies that can potentially be used List <Movie> ValidMovies = Movies.Where(v => { if (!Filter && v.UnselectFilter && v.UnselectManual == null) { return(false); } if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual) { return(false); } if (v.OptionsCTF == null) { return(false); } return(true); }).ToList(); List <string> ValidMovieNames = ValidMovies.Select(m => m.RootName).ToList(); if (ValidMovies.Count == 0) { await Dispatcher.Invoke(async() => { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", "No items were found to extract particles from.\n" + "Please make sure the names match, estimate the CTF for all items, and review your filtering thresholds."); }); } #endregion #region Read table and intersect its micrograph set with valid movies Star TableIn; if (Options.Tasks.InputOnePerItem) { List <Star> Tables = new List <Star>(); foreach (var item in Movies) { string StarPath = InputFolder + item.RootName + InputSuffix + ".star"; if (File.Exists(StarPath)) { Star TableItem = new Star(StarPath); if (!TableItem.HasColumn("rlnMicrographName")) { TableItem.AddColumn("rlnMicrographName", item.Name); } if (item.PickingThresholds.ContainsKey(InputSuffix) && TableItem.HasColumn("rlnAutopickFigureOfMerit")) { float Threshold = (float)item.PickingThresholds[InputSuffix]; float[] Scores = TableItem.GetColumn("rlnAutopickFigureOfMerit").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray(); int[] InvalidRows = Helper.ArrayOfSequence(0, TableItem.RowCount, 1).Where(i => Scores[i] < Threshold).ToArray(); TableItem.RemoveRows(InvalidRows); } Tables.Add(TableItem); } } TableIn = new Star(Tables.ToArray()); } else { TableIn = new Star(ImportPath); } if (!TableIn.HasColumn("rlnMicrographName")) { throw new Exception("Couldn't find rlnMicrographName column."); } if (!TableIn.HasColumn("rlnCoordinateX")) { throw new Exception("Couldn't find rlnCoordinateX column."); } if (!TableIn.HasColumn("rlnCoordinateY")) { throw new Exception("Couldn't find rlnCoordinateY column."); } Dictionary <string, List <int> > Groups = new Dictionary <string, List <int> >(); { string[] ColumnMicNames = TableIn.GetColumn("rlnMicrographName"); for (int r = 0; r < ColumnMicNames.Length; r++) { if (!Groups.ContainsKey(ColumnMicNames[r])) { Groups.Add(ColumnMicNames[r], new List <int>()); } Groups[ColumnMicNames[r]].Add(r); } Groups = Groups.ToDictionary(group => Helper.PathToName(group.Key), group => group.Value); Groups = Groups.Where(group => ValidMovieNames.Contains(group.Key)).ToDictionary(group => group.Key, group => group.Value); } bool[] RowsIncluded = new bool[TableIn.RowCount]; foreach (var group in Groups) { foreach (var r in group.Value) { RowsIncluded[r] = true; } } List <int> RowsNotIncluded = new List <int>(); for (int r = 0; r < RowsIncluded.Length; r++) { if (!RowsIncluded[r]) { RowsNotIncluded.Add(r); } } ValidMovies = ValidMovies.Where(v => Groups.ContainsKey(v.RootName)).ToList(); if (ValidMovies.Count == 0) // Exit if there is nothing to export, otherwise errors will be thrown below { return; } #endregion #region Make sure all columns are there if (!TableIn.HasColumn("rlnMagnification")) { TableIn.AddColumn("rlnMagnification", "10000.0"); } else { TableIn.SetColumn("rlnMagnification", Helper.ArrayOfConstant("10000.0", TableIn.RowCount)); } if (!TableIn.HasColumn("rlnDetectorPixelSize")) { TableIn.AddColumn("rlnDetectorPixelSize", Options.GetProcessingParticleExport().BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture)); } else { TableIn.SetColumn("rlnDetectorPixelSize", Helper.ArrayOfConstant(Options.GetProcessingParticleExport().BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture), TableIn.RowCount)); } if (!TableIn.HasColumn("rlnVoltage")) { TableIn.AddColumn("rlnVoltage", "300.0"); } if (!TableIn.HasColumn("rlnSphericalAberration")) { TableIn.AddColumn("rlnSphericalAberration", "2.7"); } if (!TableIn.HasColumn("rlnAmplitudeContrast")) { TableIn.AddColumn("rlnAmplitudeContrast", "0.07"); } if (!TableIn.HasColumn("rlnPhaseShift")) { TableIn.AddColumn("rlnPhaseShift", "0.0"); } if (!TableIn.HasColumn("rlnDefocusU")) { TableIn.AddColumn("rlnDefocusU", "0.0"); } if (!TableIn.HasColumn("rlnDefocusV")) { TableIn.AddColumn("rlnDefocusV", "0.0"); } if (!TableIn.HasColumn("rlnDefocusAngle")) { TableIn.AddColumn("rlnDefocusAngle", "0.0"); } if (!TableIn.HasColumn("rlnCtfMaxResolution")) { TableIn.AddColumn("rlnCtfMaxResolution", "999.0"); } if (!TableIn.HasColumn("rlnImageName")) { TableIn.AddColumn("rlnImageName", "None"); } if (!TableIn.HasColumn("rlnMicrographName")) { TableIn.AddColumn("rlnMicrographName", "None"); } #endregion int NDevices = GPU.GetDeviceCount(); List <int> UsedDevices = Options.MainWindow.GetDeviceList(); List <int> UsedDeviceProcesses = Helper.Combine(Helper.ArrayOfFunction(i => UsedDevices.Select(d => d + i *NDevices).ToArray(), MainWindow.GlobalOptions.ProcessesPerDevice)).ToList(); if (IsCanceled) { return; } #region Create worker processes WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount() * MainWindow.GlobalOptions.ProcessesPerDevice]; foreach (var gpuID in UsedDeviceProcesses) { Workers[gpuID] = new WorkerWrapper(gpuID); Workers[gpuID].SetHeaderlessParams(new int2(Options.Import.HeaderlessWidth, Options.Import.HeaderlessHeight), Options.Import.HeaderlessOffset, Options.Import.HeaderlessType); if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain) { Workers[gpuID].LoadGainRef(Options.Import.CorrectGain ? Options.Import.GainPath : "", Options.Import.GainFlipX, Options.Import.GainFlipY, Options.Import.GainTranspose, Options.Import.CorrectDefects ? Options.Import.DefectsPath : ""); } } #endregion #region Load gain reference if needed //Image[] ImageGain = new Image[NDevices]; //if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain && File.Exists(Options.Import.GainPath)) // for (int d = 0; d < NDevices; d++) // { // GPU.SetDevice(d); // ImageGain[d] = MainWindow.LoadAndPrepareGainReference(); // } #endregion bool Overwrite = true; if (DoAverage || DoStack) { foreach (var movie in ValidMovies) { bool FileExists = File.Exists((DoAverage ? movie.ParticlesDir : movie.ParticleMoviesDir) + movie.RootName + Options.Tasks.OutputSuffix + ".mrcs"); if (FileExists) { await Dispatcher.Invoke(async() => { var DialogResult = await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Some particle files already exist. Overwrite them?", "", MessageDialogStyle.AffirmativeAndNegative, new MetroDialogSettings() { AffirmativeButtonText = "Yes", NegativeButtonText = "No" }); if (DialogResult == MessageDialogResult.Negative) { Overwrite = false; } }); break; } } } Star TableOut = null; { Dictionary <string, Star> MicrographTables = new Dictionary <string, Star>(); #region Get coordinates float[] PosX = TableIn.GetColumn("rlnCoordinateX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray(); float[] PosY = TableIn.GetColumn("rlnCoordinateY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray(); float[] ShiftX = TableIn.HasColumn("rlnOriginX") ? TableIn.GetColumn("rlnOriginX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount]; float[] ShiftY = TableIn.HasColumn("rlnOriginY") ? TableIn.GetColumn("rlnOriginY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount]; for (int r = 0; r < TableIn.RowCount; r++) { PosX[r] -= ShiftX[r]; PosY[r] -= ShiftY[r]; } if (TableIn.HasColumn("rlnOriginX")) { TableIn.RemoveColumn("rlnOriginX"); } if (TableIn.HasColumn("rlnOriginY")) { TableIn.RemoveColumn("rlnOriginY"); } #endregion Dispatcher.Invoke(() => ProgressWrite.MaxValue = ValidMovies.Count); Helper.ForEachGPU(ValidMovies, (movie, gpuID) => { if (IsCanceled) { return; } Stopwatch ItemTime = new Stopwatch(); ItemTime.Start(); MapHeader OriginalHeader = MapHeader.ReadFromFile(movie.Path); #region Set up export options ProcessingOptionsParticlesExport ExportOptions = Options.GetProcessingParticleExport(); ExportOptions.DoAverage = DoAverage; ExportOptions.DoDenoisingPairs = DoDenoisingPairs; ExportOptions.DoStack = DoStack; ExportOptions.Invert = Invert; ExportOptions.Normalize = Normalize; ExportOptions.PreflipPhases = Preflip; ExportOptions.Dimensions = OriginalHeader.Dimensions.MultXY((float)Options.PixelSizeMean); ExportOptions.BoxSize = BoxSize; ExportOptions.Diameter = NormDiameter; #endregion bool FileExists = File.Exists((DoAverage ? movie.ParticlesDir : movie.ParticleMoviesDir) + movie.RootName + ExportOptions.Suffix + ".mrcs"); #region Load and prepare original movie //Image OriginalStack = null; decimal ScaleFactor = 1M / (decimal)Math.Pow(2, (double)ExportOptions.BinTimes); if (!DoOnlyStar && (!FileExists || Overwrite)) { Workers[gpuID].LoadStack(movie.Path, ScaleFactor, ExportOptions.EERGroupFrames); } //MainWindow.LoadAndPrepareHeaderAndMap(movie.Path, ImageGain[gpuID], ScaleFactor, out OriginalHeader, out OriginalStack); if (IsCanceled) { //foreach (Image gain in ImageGain) // gain?.Dispose(); //OriginalStack?.Dispose(); return; } #endregion #region Figure out relative or absolute path to particle stack string PathStack = (ExportOptions.DoStack ? movie.ParticleMoviesDir : movie.ParticlesDir) + movie.RootName + ExportOptions.Suffix + ".mrcs"; string PathMicrograph = movie.Path; if (Relative) { Uri UriStar = new Uri(ExportPath); PathStack = UriStar.MakeRelativeUri(new Uri(PathStack)).ToString(); PathMicrograph = UriStar.MakeRelativeUri(new Uri(PathMicrograph)).ToString(); } #endregion #region Update row values List <int> GroupRows = Groups[movie.RootName]; List <float2> Positions = new List <float2>(); float Astigmatism = (float)movie.CTF.DefocusDelta / 2; float PhaseShift = movie.OptionsCTF.DoPhase ? movie.GridCTFPhase.GetInterpolated(new float3(0.5f)) * 180 : 0; int ImageNameIndex = TableIn.GetColumnID("rlnImageName"); foreach (var r in GroupRows) { float3 Position = new float3(PosX[r] * AngPix / ExportOptions.Dimensions.X, PosY[r] * AngPix / ExportOptions.Dimensions.Y, 0.5f); float LocalDefocus = movie.GridCTFDefocus.GetInterpolated(Position); TableIn.SetRowValue(r, "rlnDefocusU", ((LocalDefocus + Astigmatism) * 1e4f).ToString("F1", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnDefocusV", ((LocalDefocus - Astigmatism) * 1e4f).ToString("F1", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnDefocusAngle", movie.CTF.DefocusAngle.ToString("F1", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnVoltage", movie.CTF.Voltage.ToString("F1", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnSphericalAberration", movie.CTF.Cs.ToString("F4", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnAmplitudeContrast", movie.CTF.Amplitude.ToString("F3", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnPhaseShift", PhaseShift.ToString("F1", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnCtfMaxResolution", movie.CTFResolutionEstimate.ToString("F1", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnCoordinateX", (PosX[r] * AngPix / (float)ExportOptions.BinnedPixelSizeMean).ToString("F2", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnCoordinateY", (PosY[r] * AngPix / (float)ExportOptions.BinnedPixelSizeMean).ToString("F2", CultureInfo.InvariantCulture)); TableIn.SetRowValue(r, "rlnImageName", PathStack); TableIn.SetRowValue(r, "rlnMicrographName", PathMicrograph); Positions.Add(new float2(PosX[r] * AngPix, PosY[r] * AngPix)); } #endregion #region Populate micrograph table with rows for all exported particles Star MicrographTable = new Star(TableIn.GetColumnNames()); int StackDepth = (ExportOptions.DoAverage || ExportOptions.DoDenoisingPairs || DoOnlyStar) ? 1 : (OriginalHeader.Dimensions.Z - ExportOptions.SkipFirstN - ExportOptions.SkipLastN) / ExportOptions.StackGroupSize; int pi = 0; for (int i = 0; i < StackDepth; i++) { foreach (var r in GroupRows) { List <string> Row = TableIn.GetRow(r).ToList(); Row[ImageNameIndex] = (++pi).ToString("D7") + "@" + Row[ImageNameIndex]; MicrographTable.AddRow(Row); } } #endregion #region Finally, process and export the actual particles if (!DoOnlyStar && (!FileExists || Overwrite)) { Workers[gpuID].MovieExportParticles(movie.Path, ExportOptions, Positions.ToArray()); //movie.ExportParticles(OriginalStack, Positions.ToArray(), ExportOptions); //OriginalStack.Dispose(); } #endregion #region Add this micrograph's table to global collection, update remaining time estimate lock (MicrographTables) { MicrographTables.Add(movie.RootName, MicrographTable); Timings.Add(ItemTime.ElapsedMilliseconds / (float)NDevices); int MsRemaining = (int)(MathHelper.Mean(Timings) * (ValidMovies.Count - MicrographTables.Count)); TimeSpan SpanRemaining = new TimeSpan(0, 0, 0, 0, MsRemaining); Dispatcher.Invoke(() => TextRemaining.Text = SpanRemaining.ToString((int)SpanRemaining.TotalHours > 0 ? @"hh\:mm\:ss" : @"mm\:ss")); Dispatcher.Invoke(() => { ProgressWrite.IsIndeterminate = false; ProgressWrite.Value = MicrographTables.Count; }); } #endregion }, 1, UsedDeviceProcesses); if (MicrographTables.Count > 0) { TableOut = new Star(MicrographTables.Values.ToArray()); } } Thread.Sleep(10000); // Writing out particle stacks is async, so if workers are killed immediately they may not write out everything foreach (var worker in Workers) { worker?.Dispose(); } //foreach (Image gain in ImageGain) // gain?.Dispose(); if (IsCanceled) { return; } TableOut.Save(ExportPath); }); DataContext = null; Close?.Invoke(); }
private async void ButtonAddLocal_OnClick(object sender, RoutedEventArgs e) { System.Windows.Forms.OpenFileDialog OpenDialog = new System.Windows.Forms.OpenFileDialog(); OpenDialog.Filter = "Warp Folder Settings|*.settings|Warp Data Source|*.source"; System.Windows.Forms.DialogResult OpenResult = OpenDialog.ShowDialog(); if (OpenResult.ToString() == "OK") { FileInfo Info = new FileInfo(OpenDialog.FileName); #region Check if user wants to use an existing source file instead //if (Info.Extension.ToLower() == ".settings" && // File.Exists(OpenDialog.FileName.Substring(0, OpenDialog.FileName.LastIndexOf(".")) + ".source")) //{ // var Result = await ((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Similar source file found", // $"Would you like to use {Helper.PathToName(Info.Name) + ".source"} instead?", // MessageDialogStyle.AffirmativeAndNegativeAndSingleAuxiliary, // new MetroDialogSettings // { // AffirmativeButtonText = "Use existing source", // NegativeButtonText = "Replace with new source from settings", // FirstAuxiliaryButtonText = "Cancel" // }); // if (Result == MessageDialogResult.FirstAuxiliary) // Cancel // { // return; // } // else if (Result == MessageDialogResult.Affirmative) // Use existing .source // { // OpenDialog.FileName = OpenDialog.FileName.Substring(0, OpenDialog.FileName.LastIndexOf(".")) + ".source"; // Info = new FileInfo(OpenDialog.FileName); // } //} #endregion if (Info.Extension.ToLower() == ".settings") { #region Load preprocessing options Warp.Options Options = new Warp.Options(); try { Options.Load(OpenDialog.FileName); } catch { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", $"An error was encountered when reading {Info.Name}.", MessageDialogStyle.Affirmative); return; } #endregion #region Load items with metadata string FileExtension = Options.Import.Extension; var AvailableFiles = Directory.EnumerateFiles(Info.DirectoryName, FileExtension).ToArray(); Movie[] Items = new Movie[AvailableFiles.Length]; string[] MatchingStarNames = null; if (FileExtension != "*.tomostar" && AvailableFiles.Length > 0) { Movie First = new Movie(AvailableFiles[0]); MatchingStarNames = Directory.EnumerateFiles(First.MatchingDir, "*.star").ToArray(); } { var ProgressDialog = await((MainWindow)Application.Current.MainWindow).ShowProgressAsync("Loading metadata...", ""); ProgressDialog.Maximum = AvailableFiles.Count(); await Task.Run(() => { int Done = 0; Parallel.For(0, AvailableFiles.Length, i => { string file = AvailableFiles[i]; string XmlPath = file.Substring(0, file.LastIndexOf(".")) + ".xml"; if (File.Exists(XmlPath)) { Items[i] = (FileExtension == "*.tomostar" ? new TiltSeries(file) : new Movie(file, MatchingStarNames)); } lock (Items) ProgressDialog.SetProgress(++Done); }); }); await ProgressDialog.CloseAsync(); } if (Items.Length == 0) { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", $"No micrographs or tilt series found to match these settings.", MessageDialogStyle.Affirmative); return; } #endregion #region Figure out filtering status #region Astigmatism statistics Movie[] ItemsWithCTF = Items.Where(v => v.OptionsCTF != null && v.CTF != null).ToArray(); List <float2> AstigmatismPoints = new List <float2>(ItemsWithCTF.Length); foreach (var item in ItemsWithCTF) { AstigmatismPoints.Add(new float2((float)Math.Cos((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta, (float)Math.Sin((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta)); } float2 AstigmatismMean = new float2(); float AstigmatismStd = 0.1f; if (AstigmatismPoints.Count > 0) { AstigmatismMean = new float2(); foreach (var point in AstigmatismPoints) { AstigmatismMean += point; } AstigmatismMean /= AstigmatismPoints.Count; AstigmatismStd = 0; foreach (var point in AstigmatismPoints) { AstigmatismStd += (point - AstigmatismMean).LengthSq(); } AstigmatismStd = (float)Math.Max(1e-4, Math.Sqrt(AstigmatismStd / AstigmatismPoints.Count)); } #endregion int[] ParticleCounts = Items.Select(item => item.GetParticleCount(Options.Filter.ParticlesSuffix)).ToArray(); bool HaveParticles = ParticleCounts.Any(v => v > 0); foreach (var item in Items) { bool FilterStatus = true; if (item.OptionsCTF != null) { FilterStatus &= item.CTF.Defocus >= Options.Filter.DefocusMin && item.CTF.Defocus <= Options.Filter.DefocusMax; float AstigmatismDeviation = (new float2((float)Math.Cos((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta, (float)Math.Sin((float)item.CTF.DefocusAngle * 2 * Helper.ToRad) * (float)item.CTF.DefocusDelta) - AstigmatismMean).Length() / AstigmatismStd; FilterStatus &= AstigmatismDeviation <= (float)Options.Filter.AstigmatismMax; FilterStatus &= item.CTFResolutionEstimate <= Options.Filter.ResolutionMax; FilterStatus &= item.CTF.PhaseShift >= Options.Filter.PhaseMin && item.CTF.PhaseShift <= Options.Filter.PhaseMax; FilterStatus &= item.GetParticleCount(Options.Filter.ParticlesSuffix) >= Options.Filter.ParticlesMin; } if (item.OptionsMovement != null) { FilterStatus &= item.MeanFrameMovement <= Options.Filter.MotionMax; } item.UnselectFilter = !FilterStatus; } ProcessingOptionsMovieCTF OptionsCTF = Options.GetProcessingMovieCTF(); ProcessingOptionsMovieMovement OptionsMovement = Options.GetProcessingMovieMovement(); ProcessingOptionsBoxNet OptionsBoxNet = Options.GetProcessingBoxNet(); ProcessingOptionsMovieExport OptionsExport = Options.GetProcessingMovieExport(); List <Movie> ItemsProcessed = new List <Movie>(); List <Movie> ItemsFilteredOut = new List <Movie>(); List <Movie> ItemsUnselected = new List <Movie>(); foreach (Movie item in Items) { ProcessingStatus Status = GetMovieProcessingStatus(item, OptionsCTF, OptionsMovement, OptionsBoxNet, OptionsExport, Options); if (Status == ProcessingStatus.Processed || (Status == ProcessingStatus.Outdated && !item.UnselectFilter)) { ItemsProcessed.Add(item); } else if (Status == ProcessingStatus.FilteredOut || (Status == ProcessingStatus.Outdated && item.UnselectFilter)) { ItemsFilteredOut.Add(item); } else if (Status == ProcessingStatus.LeaveOut) { ItemsUnselected.Add(item); } } #endregion #region Figure out how many frames/tilts there are to use int UsableFrames = 1; if (Items[0].GetType() == typeof(Movie)) { UsableFrames = MapHeader.ReadFromFile(Items[0].Path).Dimensions.Z; } else { UsableFrames = ((TiltSeries)Items[0]).NTilts; } #endregion #region Show dialog CustomDialog Dialog = new CustomDialog(); Dialog.HorizontalContentAlignment = HorizontalAlignment.Center; DialogCreateSourceFromSettings DialogContent = new DialogCreateSourceFromSettings(); DialogContent.TextTitle.Text = $"Create data source from\n{Info.Name}"; DialogContent.StatsSeriesStatusProcessed.Values = new ChartValues <ObservableValue> { new ObservableValue(ItemsProcessed.Count) }; DialogContent.StatsSeriesStatusUnfiltered.Values = new ChartValues <ObservableValue> { new ObservableValue(ItemsFilteredOut.Count) }; DialogContent.StatsSeriesStatusUnselected.Values = new ChartValues <ObservableValue> { new ObservableValue(ItemsUnselected.Count) }; DialogContent.SliderFrames.Value = UsableFrames; DialogContent.SliderFrames.MaxValue = UsableFrames; DialogContent.Close += () => { ((MainWindow)Application.Current.MainWindow).HideMetroDialogAsync(Dialog); }; DialogContent.Create += async() => { #region Create source metadata and check if one with the same path already exists DataSource NewSource = new DataSource { PixelSizeX = Options.PixelSizeX, PixelSizeY = Options.PixelSizeY, PixelSizeAngle = Options.PixelSizeAngle, DimensionsX = Options.Tomo.DimensionsX, DimensionsY = Options.Tomo.DimensionsY, DimensionsZ = Options.Tomo.DimensionsZ, FrameLimit = (int)DialogContent.SliderFrames.Value, GainPath = Options.Import.CorrectGain ? Options.Import.GainPath : "", GainFlipX = Options.Import.GainFlipX, GainFlipY = Options.Import.GainFlipY, GainTranspose = Options.Import.GainTranspose, DosePerAngstromFrame = Options.Import.DosePerAngstromFrame, Name = DialogContent.TextSourceName.Text, Path = Info.DirectoryName + "\\" + Helper.RemoveInvalidChars(DialogContent.TextSourceName.Text) + ".source" }; if (Population.Sources.Any(s => s.Path == NewSource.Path)) { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", $"{Helper.PathToNameWithExtension(NewSource.Path)} already exists in this population. Please choose a different name.", MessageDialogStyle.Affirmative); return; } await((MainWindow)Application.Current.MainWindow).HideMetroDialogAsync(Dialog); #endregion #region Add all items and their data hashes List <Movie> AllItems = new List <Movie>(ItemsProcessed); if ((bool)DialogContent.CheckFilter.IsChecked) { AllItems.AddRange(ItemsFilteredOut); } if ((bool)DialogContent.CheckManual.IsChecked) { AllItems.AddRange(ItemsUnselected); } if (AllItems.Count == 0) { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", $"No micrographs or tilt series found to match these settings.", MessageDialogStyle.Affirmative); return; } { var ProgressDialog = await((MainWindow)Application.Current.MainWindow).ShowProgressAsync("Calculating data hashes...", ""); ProgressDialog.Maximum = AllItems.Count; await Task.Run(() => { int Done = 0; foreach (var item in AllItems) { NewSource.Files.Add(item.GetDataHash(), item.Name); ProgressDialog.SetProgress(++Done); } }); await ProgressDialog.CloseAsync(); } #endregion #region Check for overlapping hashes string[] Overlapping = Helper.Combine(Population.Sources.Select(s => s.Files.Where(f => NewSource.Files.ContainsKey(f.Key)).Select(f => f.Value).ToArray())); if (Overlapping.Length > 0) { string Offenders = ""; for (int o = 0; o < Math.Min(5, Overlapping.Length); o++) { Offenders += "\n" + Overlapping[o]; } if (Overlapping.Length > 5) { Offenders += $"\n... and {Overlapping.Length - 5} more."; } await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", "The new source contains files that are already used in this population:" + Offenders, MessageDialogStyle.Affirmative); return; } #endregion { var ProgressDialog = await((MainWindow)Application.Current.MainWindow).ShowProgressAsync("Committing initial version...", ""); ProgressDialog.Maximum = AllItems.Count; await Task.Run(() => { NewSource.Commit(); }); await ProgressDialog.CloseAsync(); } Population.Sources.Add(NewSource); UpdateGridItems(); }; Dialog.Content = DialogContent; await((MainWindow)Application.Current.MainWindow).ShowMetroDialogAsync(Dialog, new MetroDialogSettings() { }); #endregion } else { try { if (Population.Sources.Any(s => s.Path == OpenDialog.FileName)) { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", "This data source is already part of this population.", MessageDialogStyle.Affirmative); return; } DataSource NewSource = DataSource.FromFile(OpenDialog.FileName); Population.Sources.Add(NewSource); UpdateGridItems(); } catch { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", $"An error was encountered when reading {Info.Name}.", MessageDialogStyle.Affirmative); return; } } } }
private async void ButtonWrite_OnClick(object sender, RoutedEventArgs e) { string Suffix = TextSuffix.Text; float ScaleFactorCoords = (float)(Options.Tasks.InputPixelSize / Options.BinnedPixelSizeMean); float ScaleFactorShifts = (float)(Options.Tasks.InputShiftPixelSize / Options.BinnedPixelSizeMean); ProgressWrite.Visibility = Visibility.Visible; PanelButtons.Visibility = Visibility.Hidden; await Task.Run(async() => { Star TableIn = new Star(ImportPath); #region Make sure everything is there if (TableIn.RowCount == 0 || !TableIn.HasColumn("rlnCoordinateX") || !TableIn.HasColumn("rlnCoordinateY") || !TableIn.HasColumn("rlnMicrographName")) { await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie", "Please make sure the table contains data with rlnCoordinateX, rlnCoordinateY and rlnMicrographName columns."); Close?.Invoke(); return; } if (Movies.Length == 0) { Close?.Invoke(); return; } #endregion Directory.CreateDirectory(Movies[0].MatchingDir); Dispatcher.Invoke(() => ProgressWrite.Maximum = Movies.Length); int IndexMicName = TableIn.GetColumnID("rlnMicrographName"); TableIn.ModifyAllValuesInColumn("rlnMicrographName", v => Helper.PathToName(v)); int IndexCoordX = TableIn.GetColumnID("rlnCoordinateX"); int IndexCoordY = TableIn.GetColumnID("rlnCoordinateY"); #region Subtract origin from coordinates int IndexOriginX = TableIn.GetColumnID("rlnOriginX"); if (IndexOriginX >= 0) { TableIn.ModifyAllValuesInColumn("rlnCoordinateX", (v, r) => (float.Parse(v) * ScaleFactorCoords - float.Parse(TableIn.GetRowValue(r, IndexOriginX)) * ScaleFactorShifts).ToString("F4")); TableIn.SetColumn("rlnOriginX", Helper.ArrayOfConstant("0.0", TableIn.RowCount)); } else { TableIn.ModifyAllValuesInColumn("rlnCoordinateX", v => (float.Parse(v) * ScaleFactorCoords).ToString("F4")); } int IndexOriginY = TableIn.GetColumnID("rlnOriginY"); if (IndexOriginY >= 0) { TableIn.ModifyAllValuesInColumn("rlnCoordinateY", (v, r) => (float.Parse(v) * ScaleFactorCoords - float.Parse(TableIn.GetRowValue(r, IndexOriginY)) * ScaleFactorShifts).ToString("F4")); TableIn.SetColumn("rlnOriginY", Helper.ArrayOfConstant("0.0", TableIn.RowCount)); } else { TableIn.ModifyAllValuesInColumn("rlnCoordinateY", v => (float.Parse(v) * ScaleFactorCoords).ToString("F4")); } #endregion var GroupedRows = TableIn.GetAllRows().GroupBy(row => row[IndexMicName]).ToDictionary(g => g.Key, g => g.ToList()); foreach (var movie in Movies) { if (GroupedRows.ContainsKey(movie.RootName)) { int3 Dims = MapHeader.ReadFromFile(movie.Path, new int2(Options.Import.HeaderlessWidth, Options.Import.HeaderlessHeight), Options.Import.HeaderlessOffset, ImageFormatsHelper.StringToType(Options.Import.HeaderlessType)).Dimensions; float3 BinnedDims = new float3(Dims) * (float)Options.PixelSizeMean / (float)Options.BinnedPixelSizeMean; List <List <string> > Rows = GroupedRows[movie.RootName]; foreach (var row in Rows) { row[IndexMicName] = movie.Name; float CoordX = float.Parse(row[IndexCoordX]); if (Options.Tasks.InputFlipX) { CoordX = BinnedDims.X - CoordX; } float CoordY = float.Parse(row[IndexCoordY]); if (Options.Tasks.InputFlipY) { CoordY = BinnedDims.Y - CoordY; } row[IndexCoordX] = CoordX.ToString("F4"); row[IndexCoordY] = CoordY.ToString("F4"); } Star TableOut = new Star(TableIn.GetColumnNames()); foreach (var row in Rows) { TableOut.AddRow(row); } TableOut.Save(movie.MatchingDir + movie.RootName + "_" + Suffix + ".star"); movie.UpdateParticleCount("_" + Suffix); } Dispatcher.Invoke(() => ProgressWrite.Value++); } }); Close?.Invoke(); }
private async void ButtonCreateStacks_Click(object sender, RoutedEventArgs e) { var ProgressDialog = await Options.MainWindow.ShowProgressAsync("Writing data...", ""); ObservableCollection <ParsedEntry> _ParsedEntries = new ObservableCollection <ParsedEntry>(ParsedEntries.Where(v => v.DoImport)); await Task.Run(() => { Image StackBuffer = null; int MaxTilts = 0; int idone = 0; foreach (var entry in _ParsedEntries) { Dispatcher.Invoke(() => ProgressDialog.SetTitle($"Writing stack for {entry.Name}...")); Directory.CreateDirectory(Path.Combine(Options.Import.Folder, "imod", entry.Name)); Movie[] Movies = Helper.ArrayOfFunction(i => new Movie(Path.Combine(Options.Import.Folder, entry.Table.GetRowValue(i, "wrpMovieName"))), entry.Table.RowCount); MapHeader Header = MapHeader.ReadFromFile(Movies[0].AveragePath); int3 StackDims = new int3(Header.Dimensions.X, Header.Dimensions.Y, Movies.Length); if (StackBuffer == null) { StackBuffer = new Image(StackDims); } else if (StackBuffer.Dims != StackDims) { if (MaxTilts < StackDims.Z || StackBuffer.Dims.Slice() != StackDims.Slice()) { StackBuffer.Dispose(); StackBuffer = new Image(StackDims); } else { StackBuffer.Dims = StackDims; } } MaxTilts = StackDims.Z; float[][] StackData = StackBuffer.GetHost(Intent.Write); for (int i = 0; i < Movies.Length; i++) { IOHelper.ReadMapFloatPatient(50, 500, Movies[i].AveragePath, new int2(1), 0, typeof(float), new[] { 0 }, null, new[] { StackData[i] }); } //HeaderMRC StackHeader = new HeaderMRC(); //StackHeader.ImodTilt = entry.TiltAngles; //StackHeader.ImodRotation = entry.Rotation; //StackHeader.PixelSize = Header.PixelSize; StackBuffer.WriteMRC(Path.Combine(Options.Import.Folder, "imod", entry.Name, entry.Name + ".st"), Header.PixelSize.X, true, null); using (TextWriter Writer = File.CreateText(Path.Combine(Options.Import.Folder, "imod", entry.Name, entry.Name + ".rawtlt"))) { foreach (var angle in entry.TiltAngles) { Writer.WriteLine(angle.ToString(CultureInfo.InvariantCulture)); } } //HeaderMRC H = (HeaderMRC)MapHeader.ReadFromFile(Path.Combine(Options.Import.Folder, "imod", entry.Name, entry.Name + ".st")); Dispatcher.Invoke(() => ProgressDialog.SetProgress(++idone / (double)_ParsedEntries.Count)); } }); await ProgressDialog.CloseAsync(); Close?.Invoke(); }
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int layer = -1, Stream stream = null) { MapHeader Header = null; Type ValueType = null; float[][] Data; if (MapHeader.GetHeaderType(path) != typeof(HeaderTiff)) { using (BinaryReader Reader = isBigEndian ? new BinaryReaderBE(OpenWithBigBuffer(path)) : new BinaryReader(OpenWithBigBuffer(path))) { Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType); ValueType = Header.GetValueType(); Data = new float[layer < 0 ? Header.Dimensions.Z : 1][]; for (int z = 0; z < Data.Length; z++) { if (layer >= 0) { Reader.BaseStream.Seek((long)Header.Dimensions.ElementsSlice() * (long)ImageFormatsHelper.SizeOf(ValueType) * layer, SeekOrigin.Current); } byte[] Bytes = Reader.ReadBytes((int)Header.Dimensions.ElementsSlice() * (int)ImageFormatsHelper.SizeOf(ValueType)); Data[z] = new float[(int)Header.Dimensions.ElementsSlice()]; if (isBigEndian) { if (ValueType == typeof(short) || ValueType == typeof(ushort)) { for (int i = 0; i < Bytes.Length / 2; i++) { Array.Reverse(Bytes, i * 2, 2); } } else if (ValueType == typeof(int) || ValueType == typeof(float)) { for (int i = 0; i < Bytes.Length / 4; i++) { Array.Reverse(Bytes, i * 4, 4); } } else if (ValueType == typeof(double)) { for (int i = 0; i < Bytes.Length / 8; i++) { Array.Reverse(Bytes, i * 8, 8); } } } unsafe { int Elements = (int)Header.Dimensions.ElementsSlice(); fixed(byte *BytesPtr = Bytes) fixed(float *DataPtr = Data[z]) { float *DataP = DataPtr; if (ValueType == typeof(byte)) { byte *BytesP = BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(short)) { short *BytesP = (short *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(ushort)) { ushort *BytesP = (ushort *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(int)) { int *BytesP = (int *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(float)) { float *BytesP = (float *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = *BytesP++; } } else if (ValueType == typeof(double)) { double *BytesP = (double *)BytesPtr; for (int i = 0; i < Elements; i++) { *DataP++ = (float)*BytesP++; } } } } } } } else { Header = MapHeader.ReadFromFile(null, path, headerlessSliceDims, headerlessOffset, headerlessType, stream); Data = ((HeaderTiff)Header).ReadData(stream, layer); } return(Data); }
private async void ButtonWrite_OnClick(object sender, RoutedEventArgs e) { System.Windows.Forms.SaveFileDialog FileDialog = new System.Windows.Forms.SaveFileDialog { Filter = "STAR Files|*.star" }; System.Windows.Forms.DialogResult Result = FileDialog.ShowDialog(); if (Result.ToString() == "OK") { ExportPath = FileDialog.FileName; } else { return; } bool Relative = (bool)CheckRelative.IsChecked; bool Filter = (bool)CheckFilter.IsChecked; bool Manual = (bool)CheckManual.IsChecked; ProgressWrite.Visibility = Visibility.Visible; PanelButtons.Visibility = Visibility.Hidden; await Task.Run(() => { List <Movie> ValidMovies = Movies.Where(v => { if (!Filter && v.UnselectFilter && v.UnselectManual == null) { return(false); } if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual) { return(false); } if (Options.Tasks.MicListMakePolishing && (v.OptionsMovement == null || v.GridMovementX.Dimensions.Z <= 1)) { return(false); } return(true); }).ToList(); string PathPrefix = "", PathPrefixAverage = "", GainPath = ""; if (ValidMovies.Count > 0 && Relative) { Uri UriStar = new Uri(ExportPath); PathPrefix = Helper.PathToFolder(UriStar.MakeRelativeUri(new Uri(ValidMovies[0].Path)).ToString()); if (PathPrefix == ValidMovies[0].Name) { PathPrefix = ""; } PathPrefixAverage = Helper.PathToFolder(UriStar.MakeRelativeUri(new Uri(ValidMovies[0].AveragePath)).ToString()); if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain) { // Since RELION can't read DM4 or flip/transpose the gain, save a transformed version as MRC. string TransformedGainPath = Helper.PathToFolder(Options.Import.GainPath) + "gain/transformed.mrc"; Directory.CreateDirectory(Helper.PathToFolder(TransformedGainPath)); Image TransformedGain = MainWindow.LoadAndPrepareGainReference(); TransformedGain.WriteMRC(TransformedGainPath, (float)Options.PixelSizeMean, true); TransformedGain.Dispose(); GainPath = UriStar.MakeRelativeUri(new Uri(TransformedGainPath)).ToString(); } } Dispatcher.Invoke(() => ProgressWrite.Maximum = ValidMovies.Count); bool IncludeCTF = ValidMovies.Any(v => v.OptionsCTF != null); bool IncludePolishing = ValidMovies.Any(v => v.OptionsMovement != null) && Options.Tasks.MicListMakePolishing; Star TableOut = new Star(new[] { "rlnMicrographName" }); if (IncludeCTF) { TableOut.AddColumn("rlnMagnification"); TableOut.AddColumn("rlnDetectorPixelSize"); TableOut.AddColumn("rlnVoltage"); TableOut.AddColumn("rlnSphericalAberration"); TableOut.AddColumn("rlnAmplitudeContrast"); TableOut.AddColumn("rlnPhaseShift"); TableOut.AddColumn("rlnDefocusU"); TableOut.AddColumn("rlnDefocusV"); TableOut.AddColumn("rlnDefocusAngle"); TableOut.AddColumn("rlnCtfMaxResolution"); TableOut.AddColumn("rlnBeamTiltX"); TableOut.AddColumn("rlnBeamTiltY"); } if (IncludePolishing) { TableOut.AddColumn("rlnMicrographMetadata"); } List <Action> TablesToWrite = new List <Action>(); if (IncludePolishing && ValidMovies.Count > 0) { Directory.CreateDirectory(ValidMovies[0].DirectoryName + "motion"); } int r = 0; Parallel.ForEach(ValidMovies, new ParallelOptions() { MaxDegreeOfParallelism = 4 }, movie => { List <string> Row = new List <string>() { PathPrefixAverage + movie.RootName + ".mrc" }; if (movie.OptionsCTF != null) { Row.AddRange(new[] { (1e4M / movie.OptionsCTF.BinnedPixelSizeMean).ToString("F1", CultureInfo.InvariantCulture), "1.0", movie.CTF.Voltage.ToString("F1", CultureInfo.InvariantCulture), movie.CTF.Cs.ToString("F4", CultureInfo.InvariantCulture), movie.CTF.Amplitude.ToString("F3", CultureInfo.InvariantCulture), (movie.CTF.PhaseShift * 180M).ToString("F1", CultureInfo.InvariantCulture), ((movie.CTF.Defocus + movie.CTF.DefocusDelta / 2) * 1e4M).ToString("F1", CultureInfo.InvariantCulture), ((movie.CTF.Defocus - movie.CTF.DefocusDelta / 2) * 1e4M).ToString("F1", CultureInfo.InvariantCulture), movie.CTF.DefocusAngle.ToString("F1", CultureInfo.InvariantCulture), movie.CTFResolutionEstimate.ToString("F1", CultureInfo.InvariantCulture), movie.CTF.BeamTilt.X.ToString("F2", CultureInfo.InvariantCulture), movie.CTF.BeamTilt.Y.ToString("F2", CultureInfo.InvariantCulture) }); } else { Row.AddRange(Helper.ArrayOfFunction(i => "0.0", 12)); } if (IncludePolishing) { Row.Add(PathPrefix + "motion/" + movie.RootName + ".star"); } lock (TableOut) TableOut.AddRow(Row); if (IncludePolishing) { MapHeader Header = MapHeader.ReadFromFile(movie.Path); List <string> HeaderNames = new List <string>() { "rlnImageSizeX", "rlnImageSizeY", "rlnImageSizeZ", "rlnMicrographMovieName", "rlnMicrographBinning", "rlnMicrographOriginalPixelSize", "rlnMicrographDoseRate", "rlnMicrographPreExposure", "rlnVoltage", "rlnMicrographStartFrame", "rlnMotionModelVersion" }; List <string> HeaderValues = new List <string>() { Header.Dimensions.X.ToString(), Header.Dimensions.Y.ToString(), Header.Dimensions.Z.ToString(), PathPrefix + movie.Name, Math.Pow(2.0, (double)Options.Import.BinTimes).ToString("F5"), Options.PixelSizeMean.ToString("F5"), Options.Import.DosePerAngstromFrame.ToString("F5"), "0", Options.CTF.Voltage.ToString(), "1", "0" }; if (!string.IsNullOrEmpty(GainPath)) { HeaderNames.Add("rlnMicrographGainName"); HeaderValues.Add(GainPath); } StarParameters ParamsTable = new StarParameters(HeaderNames.ToArray(), HeaderValues.ToArray()); float2[] MotionTrack = movie.GetMotionTrack(new float2(0.5f, 0.5f), 1).Select(v => v / (float)Options.PixelSizeMean).ToArray(); Star TrackTable = new Star(new[] { Helper.ArrayOfSequence(1, MotionTrack.Length + 1, 1).Select(v => v.ToString()).ToArray(), MotionTrack.Select(v => (-v.X).ToString("F5")).ToArray(), MotionTrack.Select(v => (-v.Y).ToString("F5")).ToArray() }, new[] { "rlnMicrographFrameNumber", "rlnMicrographShiftX", "rlnMicrographShiftY" }); Star.SaveMultitable(movie.DirectoryName + "motion/" + movie.RootName + ".star", new Dictionary <string, Star>() { { "general", ParamsTable }, { "global_shift", TrackTable }, }); } lock (this) { r++; if (r % 10 == 0 || r == ValidMovies.Count) { Dispatcher.Invoke(() => ProgressWrite.Value = r); } } }); TableOut.Save(ExportPath); }); Close?.Invoke(); }
public void UpdateDepiction() { if (DepictionMesh != null) { DepictionMesh.Dispose(); DepictionMesh = null; } foreach (var point in Points) { point.DropDepictionMesh(); } //if (Points.Count == 0) //return; if (Depiction == PointDepiction.Mesh && File.Exists(DepictionMeshPath)) { FileInfo Info = new FileInfo(DepictionMeshPath); if (Info.Extension.ToLower().Contains("mrc")) { HeaderMRC VolumeHeader = (HeaderMRC)MapHeader.ReadFromFile(DepictionMeshPath); float[] VolumeData = IOHelper.ReadSmallMapFloat(DepictionMeshPath, new int2(1, 1), 0, typeof(float)); Mesh NewMesh = Mesh.FromVolume(VolumeData, VolumeHeader.Dimensions, VolumeHeader.Pixelsize.X, (float)DepictionMeshLevel); NewMesh.UsedComponents = MeshVertexComponents.Position | MeshVertexComponents.Normal; NewMesh.GLContext = MainWindow.Options.Viewport.GetControl(); NewMesh.UpdateBuffers(); _DepictionMesh = NewMesh; } else if (Info.Extension.ToLower().Contains("obj")) { Mesh NewMesh = Mesh.FromOBJ(DepictionMeshPath, true); NewMesh.UsedComponents = MeshVertexComponents.Position | MeshVertexComponents.Normal; NewMesh.GLContext = MainWindow.Options.Viewport.GetControl(); NewMesh.UpdateBuffers(); _DepictionMesh = NewMesh; } } else if (Depiction == PointDepiction.LocalSurface && MainWindow.Options.Membrane.TomogramTexture != null) { int3 DimsExtract = new int3((int)Size + 2, (int)Size + 2, (int)Size + 2); Parallel.ForEach(Points, point => { Vector3 TomoPos = (point.Position - MainWindow.Options.Membrane.TomogramTexture.Offset) / MainWindow.Options.PixelScale.X; int3 TomoPosInt = new int3((int)Math.Round(TomoPos.X), (int)Math.Round(TomoPos.Y), (int)Math.Round(TomoPos.Z)); float[] LocalVol = Helper.Extract(MainWindow.Options.Membrane.TomogramTexture.OriginalData, MainWindow.Options.Membrane.TomogramTexture.Size, TomoPosInt, DimsExtract); if (DepictionLocalSurfaceInvert) { for (int i = 0; i < LocalVol.Length; i++) { LocalVol[i] = -LocalVol[i]; } } for (int z = 0; z < DimsExtract.Z; z++) { for (int y = 0; y < DimsExtract.Y; y++) { for (int x = 0; x < DimsExtract.X; x++) { if (z == 0 || y == 0 || x == 0 || z == DimsExtract.Z - 1 || y == DimsExtract.Y - 1 || x == DimsExtract.X - 1) { LocalVol[(z * DimsExtract.Y + y) * DimsExtract.X + x] = -99999; } } } } bool[] Mask = new bool[LocalVol.Length]; float Threshold = (float)DepictionLocalSurfaceLevel; if (DepictionLocalSurfaceOnlyCenter) { int MostCentralID = -1; float MostCentralDist = DimsExtract.X * DimsExtract.X; // Find most central valid pixel in the local window to start mask expansion from there. for (int z = 1; z < DimsExtract.Z - 1; z++) { int zz = z - DimsExtract.Z / 2; zz *= zz; for (int y = 1; y < DimsExtract.Y - 1; y++) { int yy = y - DimsExtract.Y / 2; yy *= yy; for (int x = 1; x < DimsExtract.X - 1; x++) { if (LocalVol[(z * DimsExtract.Y + y) * DimsExtract.X + x] >= Threshold) { int xx = x - DimsExtract.X / 2; xx *= xx; float r = xx + yy + zz; if (r < MostCentralDist) { MostCentralDist = r; MostCentralID = (z * DimsExtract.Y + y) * DimsExtract.X + x; } } } } } if (MostCentralID < 0) // Volume doesn't contain voxels above threshold { return; } Mask[MostCentralID] = true; for (int mi = 0; mi < Size / 2; mi++) { bool[] NextMask = new bool[Mask.Length]; for (int z = 1; z < DimsExtract.Z - 1; z++) { for (int y = 1; y < DimsExtract.Y - 1; y++) { for (int x = 1; x < DimsExtract.X - 1; x++) { int ID = (z * DimsExtract.Y + y) * DimsExtract.X + x; if (LocalVol[ID] >= Threshold) { if (Mask[ID] || Mask[ID + 1] || Mask[ID - 1] || Mask[ID + DimsExtract.X] || Mask[ID - DimsExtract.X] || Mask[ID + DimsExtract.Y * DimsExtract.X] || Mask[ID - DimsExtract.Y * DimsExtract.X]) { NextMask[ID] = true; } } } } } Mask = NextMask; } } else { for (int i = 0; i < Mask.Length; i++) { Mask[i] = true; } } // Apply spherical mask int Size2 = (int)(Size * Size / 4); for (int z = 1; z < DimsExtract.Z - 1; z++) { int zz = z - DimsExtract.Z / 2; zz *= zz; for (int y = 1; y < DimsExtract.Y - 1; y++) { int yy = y - DimsExtract.Y / 2; yy *= yy; for (int x = 1; x < DimsExtract.X - 1; x++) { int xx = x - DimsExtract.X / 2; xx *= xx; int r2 = xx + yy + zz; Mask[(z * DimsExtract.Y + y) * DimsExtract.X + x] &= r2 < Size2; } } } for (int i = 0; i < Mask.Length; i++) { if (!Mask[i]) { LocalVol[i] = Math.Min(LocalVol[i], Threshold - 1e-5f); } } //IOHelper.WriteMapFloat("d_extract.mrc", HeaderMRC.ReadFromFile("test_extract.mrc"), LocalVol); //IOHelper.WriteMapFloat("d_original.mrc", HeaderMRC.ReadFromFile("Tomo1L1_bin4.mrc"), MainWindow.Options.Membrane.TomogramTexture.OriginalData); point.DepictionMesh = Mesh.FromVolume(LocalVol, DimsExtract, MainWindow.Options.PixelScale.X, (float)DepictionLocalSurfaceLevel); point.DepictionMesh.UsedComponents = MeshVertexComponents.Position | MeshVertexComponents.Normal; }); foreach (var point in Points) { if (point.DepictionMesh == null) { continue; } point.DepictionMesh.GLContext = MainWindow.Options.Viewport.GetControl(); point.DepictionMesh.UpdateBuffers(); } } MainWindow.Options.Viewport.Redraw(); }
static void Main(string[] args) { Console.WriteLine("Path to stacks:"); string FolderPath = Console.ReadLine(); if (FolderPath[FolderPath.Length - 1] != '\\' || FolderPath[FolderPath.Length - 1] != '/') { FolderPath += "\\"; } Console.WriteLine("Reading stacks from " + FolderPath + "\n"); Console.WriteLine("Look for files recursively in subfolders? (y/n)"); bool DoRecursiveSearch = Console.ReadLine().ToLower().Contains("y"); Console.WriteLine($"Files will {(DoRecursiveSearch ? "" : "not ")}be searched recursively.\n"); Console.WriteLine("Output folder (leave empty if identical with input):"); string OutputPath = Console.ReadLine(); if (string.IsNullOrEmpty(OutputPath)) { OutputPath = FolderPath; } if (OutputPath[OutputPath.Length - 1] != '\\' || OutputPath[OutputPath.Length - 1] != '/') { OutputPath += "\\"; } Console.WriteLine("Writing compressed stacks to " + OutputPath + "\n"); Console.WriteLine("Compress to TIFF (c), or just move (m)? (c/m)"); bool Compress = Console.ReadLine().ToLower().Contains("c"); Console.WriteLine($"Files will be {(Compress ? "written as compressed TIFFs" : "just moved")}.\n"); string Extension = "mrc"; if (!Compress) { Console.WriteLine("What is the input file extension?"); Extension = Console.ReadLine().ToLower(); if (Extension[0] == '*') { Extension = Extension.Substring(1); } if (Extension[0] == '.') { Extension = Extension.Substring(1); } Console.WriteLine($"Using *.{Extension} as input file extension.\n"); } Console.WriteLine("Number of frames:"); string FramesString = Console.ReadLine(); int NFrames; try { NFrames = int.Parse(FramesString); Console.WriteLine("Using " + NFrames + " frames.\n"); } catch (Exception) { return; } Console.WriteLine("Delete original files after completion? (y/n)"); bool DeleteWhenDone = Console.ReadLine().ToLower().Contains("y"); Console.WriteLine($"Original files will {(DeleteWhenDone? "" : "not ")}be deleted.\n"); if (!DeleteWhenDone) { Directory.CreateDirectory(FolderPath + "original"); } Console.WriteLine("Delete superfluous gain references? (y/n)"); bool DeleteExtraGain = Console.ReadLine().ToLower().Contains("y"); Console.WriteLine($"Superfluous gain references will {(DeleteWhenDone ? "" : "not ")}be deleted.\n"); if (!DeleteWhenDone) { Directory.CreateDirectory(FolderPath + "original"); } Console.WriteLine("Number of stacks to be processed in parallel (4 is a good value):"); int NParallel = int.Parse(Console.ReadLine()); Console.WriteLine($"{NParallel} stacks will be processed in parallel.\n"); GC.AddMemoryPressure(20 * ((long)1 << 30)); List <string> HaveBeenProcessed = new List <string>(); while (true) { List <string> FrameNames = new List <string>(); List <string> GainRefNames = new List <string>(); foreach (var filename in Directory.EnumerateFiles(FolderPath, "*." + Extension, DoRecursiveSearch ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)) { try { MapHeader Header = MapHeader.ReadFromFile(filename); if (Header.Dimensions.Z != NFrames) { if (Header.Dimensions.Z == 1) { GainRefNames.Add(filename); } continue; } if (HaveBeenProcessed.Contains(filename)) { continue; } if (DeleteWhenDone) { FrameNames.Add(filename); } else if (!filename.Contains("\\original\\")) { FrameNames.Add(filename); } } catch { } } int NFiles = FrameNames.Count; if (NFiles == 0) { Thread.Sleep(1000); continue; } Console.WriteLine("Found " + NFiles + " new stacks."); Thread.Sleep(1000); SemaphoreSlim WritingSemaphore = new SemaphoreSlim(NParallel); for (int f = 0; f < NFiles; f++) { string FrameName = FrameNames[f]; MapHeader Header = MapHeader.ReadFromFilePatient(500, 100, FrameName, new int2(1), 0, typeof(float)); if (Compress) { Image StackOut = new Image(Header.Dimensions); float[][] StackOutData = StackOut.GetHost(Intent.Read); for (int n = 0; n < NFrames; n++) { Image Frame = Image.FromFilePatient(50, 100, FrameName, n); float[] FrameData = Frame.GetHost(Intent.Read)[0]; if (Compress) { for (int i = 0; i < FrameData.Length; i++) { StackOutData[n][i] = (float)Math.Max(0, Math.Min(255, Math.Round(FrameData[i]))); } } else { for (int i = 0; i < FrameData.Length; i++) { StackOutData[n][i] = FrameData[i]; } } Frame.Dispose(); Console.Write("."); } Console.WriteLine(""); HaveBeenProcessed.Add(FrameName); string RootName = Helper.PathToName(FrameName); Thread WriteThread = new Thread(() => { try { //if (Compress) StackOut.WriteTIFF(OutputPath + RootName + ".tif", 1, typeof(byte)); //else // StackOut.WriteMRC(OutputPath + RootName + ".mrc", 1, true); StackOut.Dispose(); if (DeleteWhenDone) { File.Delete(FrameName); } else { File.Move(FrameName, FolderPath + "original/" + Helper.PathToNameWithExtension(FrameName)); } } catch (Exception exc) { Console.WriteLine("ERROR: Could not write " + RootName); Console.WriteLine(exc); HaveBeenProcessed.Remove(FrameName); } WritingSemaphore.Release(); GC.Collect(); }); while (WritingSemaphore.CurrentCount < 1) { Thread.Sleep(100); } WritingSemaphore.Wait(); WriteThread.Start(); Console.WriteLine("Done reading: " + RootName); } else { bool Success = false; while (!Success) { try { string NameOut = OutputPath + Helper.PathToNameWithExtension(FrameName); if (DeleteWhenDone) { File.Move(FrameName, NameOut); } else { File.Copy(FrameName, NameOut); File.Move(FrameName, FolderPath + "original/" + Helper.PathToNameWithExtension(FrameName)); } HaveBeenProcessed.Add(FrameName); Success = true; Console.WriteLine("Done moving: " + Helper.PathToNameWithExtension(FrameName)); } catch (Exception exc) { Console.WriteLine("Something went wrong moving " + Helper.PathToNameWithExtension(FrameName) + ":\n" + exc.ToString()); } } } } if (DeleteExtraGain) { foreach (var gainRefName in GainRefNames) { File.Delete(gainRefName); } } Thread.Sleep(1000); } }
public static float[][] ReadMapFloat(BinaryReader reader, string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int[] layers = null, Stream stream = null, float[][] reuseBuffer = null) { MapHeader Header = null; Type ValueType = null; float[][] Data; BinaryReader Reader = reader; { Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType); ValueType = Header.GetValueType(); Data = reuseBuffer == null ? new float[layers == null ? Header.Dimensions.Z : layers.Length][] : reuseBuffer; int ReadBatchSize = Math.Min((int)Header.Dimensions.ElementsSlice(), 1 << 20); int ValueSize = (int)ImageFormatsHelper.SizeOf(ValueType); byte[] Bytes = new byte[ReadBatchSize * ValueSize]; long ReaderDataStart = Reader.BaseStream.Position; for (int z = 0; z < Data.Length; z++) { if (layers != null) { Reader.BaseStream.Seek(Header.Dimensions.ElementsSlice() * ImageFormatsHelper.SizeOf(ValueType) * layers[z] + ReaderDataStart, SeekOrigin.Begin); } if (reuseBuffer == null) { Data[z] = new float[(int)Header.Dimensions.ElementsSlice()]; } unsafe { fixed(byte *BytesPtr = Bytes) fixed(float *DataPtr = Data[z]) { for (int b = 0; b < (int)Header.Dimensions.ElementsSlice(); b += ReadBatchSize) { int CurBatch = Math.Min(ReadBatchSize, (int)Header.Dimensions.ElementsSlice() - b); Reader.Read(Bytes, 0, CurBatch * ValueSize); if (isBigEndian) { if (ValueType == typeof(short) || ValueType == typeof(ushort)) { for (int i = 0; i < CurBatch * ValueSize / 2; i++) { Array.Reverse(Bytes, i * 2, 2); } } else if (ValueType == typeof(int) || ValueType == typeof(float)) { for (int i = 0; i < CurBatch * ValueSize / 4; i++) { Array.Reverse(Bytes, i * 4, 4); } } else if (ValueType == typeof(double)) { for (int i = 0; i < CurBatch * ValueSize / 8; i++) { Array.Reverse(Bytes, i * 8, 8); } } } float *DataP = DataPtr + b; if (ValueType == typeof(byte)) { byte *BytesP = BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(short)) { short *BytesP = (short *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(ushort)) { ushort *BytesP = (ushort *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(int)) { int *BytesP = (int *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(float)) { float *BytesP = (float *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = *BytesP++; } } else if (ValueType == typeof(double)) { double *BytesP = (double *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } } } } } } return(Data); }
private async void Movie_PS2DChanged(object sender, EventArgs e) { try { AdjustGridVisibility(); ImagePS2D.Visibility = Visibility.Hidden; if (Movie == null) { return; } if (Movie.OptionsCTF == null || !File.Exists(Movie.PowerSpectrumPath)) { return; } // Check if 2D PS file has the PS for the currently selected tilt if (Movie.GetType() == typeof(TiltSeries)) { MapHeader Header = MapHeader.ReadFromFile(Movie.PowerSpectrumPath); if (Header.Dimensions.Z <= TiltID) { return; } } ProgressPS2D.Visibility = Visibility.Visible; Movie movie = Movie; //await Task.Delay(1000); await Task.Run(() => { ImageSource PS2D; unsafe { int Slice = -1; if (movie.GetType() == typeof(TiltSeries)) { Slice = TiltID; } MapHeader Header = MapHeader.ReadFromFile(movie.PowerSpectrumPath); float[] Data = Image.FromFile(movie.PowerSpectrumPath, Slice).GetHostContinuousCopy(); int Width = Header.Dimensions.X; int HalfWidth = Width / 2; int Height = Header.Dimensions.Y; // The usual x / 2 + 1 int RadiusMin2 = (int)(movie.OptionsCTF.RangeMin *HalfWidth); RadiusMin2 *= RadiusMin2; int RadiusMax2 = (int)(movie.OptionsCTF.RangeMax *HalfWidth); RadiusMax2 *= RadiusMax2; double Sum1 = 0; double Sum2 = 0; int Samples = 0; fixed(float *DataPtr = Data) { float *DataP = DataPtr; for (int y = 0; y < Height; y++) { for (int x = 0; x < Width; x++) { int XCentered = x - HalfWidth; int YCentered = Height - 1 - y; int Radius2 = XCentered * XCentered + YCentered * YCentered; if (Radius2 >= RadiusMin2 && Radius2 <= RadiusMax2) { Sum1 += *DataP; Sum2 += (*DataP) * (*DataP); Samples++; } DataP++; } } float Mean = (float)(Sum1 / Samples); float Std = (float)(Math.Sqrt(Samples *Sum2 - Sum1 *Sum1) / Samples); float ValueMin = Mean - 1.5f * Std; float ValueMax = Mean + 3.0f * Std; float Range = ValueMax - ValueMin; if (Range <= 0f) { PS2D = BitmapSource.Create(Width, Height, 96, 96, PixelFormats.Indexed8, BitmapPalettes.Gray256, new byte[Data.Length], Width); } else { byte[] DataBytes = new byte[Data.Length]; fixed(byte *DataBytesPtr = DataBytes) { for (int y = 0; y < Height; y++) { for (int x = 0; x < Width; x++) { DataBytesPtr[(Height - 1 - y) * Width + x] = (byte)(Math.Max(Math.Min(1f, (DataPtr[y * Width + x] - ValueMin) / Range), 0f) * 255f); } } } PS2D = BitmapSource.Create(Width, Height, 96, 96, PixelFormats.Indexed8, BitmapPalettes.Gray256, DataBytes, Width); PS2D.Freeze(); } Dispatcher.Invoke(() => { ImagePS2D.Source = PS2D; ImagePS2D.Visibility = Visibility.Visible; }); } } }); ProgressPS2D.Visibility = Visibility.Hidden; } catch { } }
private async void ButtonMatch_OnClick(object sender, RoutedEventArgs e) { bool Filter = (bool)CheckFilter.IsChecked; bool Manual = (bool)CheckManual.IsChecked; #region Get all movies that can potentially be used List <Movie> ValidMovies = Movies.Where(v => { if (!Filter && v.UnselectFilter && v.UnselectManual == null) { return(false); } if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual) { return(false); } if (v.OptionsCTF == null) { return(false); } return(true); }).ToList(); if (ValidMovies.Count == 0) { Close?.Invoke(); return; } #endregion #region Set up progress displays NParallel = Math.Min(ValidMovies.Count, GPU.GetDeviceCount()); GridSizes = Helper.ArrayOfConstant(new int3(1), NParallel); GridProgress = new int[NParallel]; GridNames = new string[NParallel]; GridControls = Helper.ArrayOfFunction(i => new FlatGridMiniature() { Width = 256 }, NParallel); GridLabels = Helper.ArrayOfFunction(i => new TextBlock { FontSize = 18, Margin = new Thickness(0, 15, 0, 0), HorizontalAlignment = HorizontalAlignment.Center }, NParallel); for (int n = 0; n < NParallel; n++) { StackPanel SP = new StackPanel { Orientation = Orientation.Vertical, Margin = new Thickness(15, 0, 15, 0) }; SP.Children.Add(GridControls[n]); SP.Children.Add(GridLabels[n]); PanelGrids.Children.Add(SP); } #endregion PanelConfiguration.Visibility = Visibility.Collapsed; PanelProgress.Visibility = Visibility.Visible; int Completed = 0; ProgressOverall.Maximum = ValidMovies.Count; await Task.Run(() => { #region Load template and make copies for all devices Image[] Template = new Image[NParallel]; { Template[0] = Image.FromFile(PathTemplate); Template[0].PixelSize = (float)Options.Tasks.TomoMatchTemplatePixel; for (int i = 0; i < Template.Length; i++) { Template[i] = Template[0].GetCopy(); } } #endregion #region Load gain reference if needed Image[] ImageGain = new Image[NParallel]; DefectModel[] DefectMap = new DefectModel[NParallel]; if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain && File.Exists(Options.Import.GainPath)) { for (int d = 0; d < NParallel; d++) { ImageGain[d] = MainWindow.LoadAndPrepareGainReference(); } } if (!string.IsNullOrEmpty(Options.Import.DefectsPath) && Options.Import.CorrectDefects && File.Exists(Options.Import.DefectsPath)) { for (int d = 0; d < NParallel; d++) { DefectMap[d] = MainWindow.LoadAndPrepareDefectMap(); } } #endregion object SyncDummy = new object(); Helper.ForEachGPU(ValidMovies, (item, gpuID) => { if (IsCanceled) { return(true); // This cancels the iterator } Dispatcher.Invoke(() => { SetGridSize(gpuID, new int3(1, 1, 1)); SetGridProgress(gpuID, 0); SetGridName(gpuID, "Loading data..."); }); ProcessingOptionsFullMatch MatchOptions = Options.GetProcessingFullMatch(); MatchOptions.TemplateName = Helper.PathToName(PathTemplate); #region Load and prepare original movie Image OriginalStack = null; decimal ScaleFactor = 1M / MatchOptions.DownsampleFactor; MapHeader OriginalHeader = MapHeader.ReadFromFile(item.Path); MainWindow.LoadAndPrepareHeaderAndMap(item.Path, ImageGain[gpuID], DefectMap[gpuID], ScaleFactor, out OriginalHeader, out OriginalStack); if (IsCanceled) { OriginalStack?.Dispose(); return(true); } #endregion item.MatchFull(OriginalStack, MatchOptions, Template[gpuID], (size, value, name) => { Dispatcher.Invoke(() => { SetGridSize(gpuID, size); SetGridProgress(gpuID, value); SetGridName(gpuID, name); }); return(IsCanceled); }); OriginalStack?.Dispose(); Dispatcher.Invoke(() => ProgressOverall.Value = ++Completed); return(false); // No need to cancel GPU ForEach iterator }, 1); }); Close?.Invoke(); }
public static float[][] ReadMapFloat(string path, int2 headerlessSliceDims, long headerlessOffset, Type headerlessType, bool isBigEndian, int[] layers = null, Stream stream = null, float[][] reuseBuffer = null) { MapHeader Header = null; Type ValueType = null; float[][] Data; if (MapHeader.GetHeaderType(path) != typeof(HeaderTiff)) { using (BinaryReader Reader = isBigEndian ? new BinaryReaderBE(OpenWithBigBuffer(path)) : new BinaryReader(OpenWithBigBuffer(path))) { Header = MapHeader.ReadFromFile(Reader, path, headerlessSliceDims, headerlessOffset, headerlessType); ValueType = Header.GetValueType(); Data = reuseBuffer == null ? new float[layers == null ? Header.Dimensions.Z : layers.Length][] : reuseBuffer; int ReadBatchSize = Math.Min((int)Header.Dimensions.ElementsSlice(), 1 << 20); int ValueSize = (int)ImageFormatsHelper.SizeOf(ValueType); byte[] Bytes = new byte[ReadBatchSize * ValueSize]; long ReaderDataStart = Reader.BaseStream.Position; for (int z = 0; z < Data.Length; z++) { if (layers != null) { Reader.BaseStream.Seek(Header.Dimensions.ElementsSlice() * ImageFormatsHelper.SizeOf(ValueType) * layers[z] + ReaderDataStart, SeekOrigin.Begin); } if (reuseBuffer == null) { Data[z] = new float[(int)Header.Dimensions.ElementsSlice()]; } unsafe { fixed(byte *BytesPtr = Bytes) fixed(float *DataPtr = Data[z]) { for (int b = 0; b < (int)Header.Dimensions.ElementsSlice(); b += ReadBatchSize) { int CurBatch = Math.Min(ReadBatchSize, (int)Header.Dimensions.ElementsSlice() - b); Reader.Read(Bytes, 0, CurBatch * ValueSize); if (isBigEndian) { if (ValueType == typeof(short) || ValueType == typeof(ushort)) { for (int i = 0; i < CurBatch * ValueSize / 2; i++) { Array.Reverse(Bytes, i * 2, 2); } } else if (ValueType == typeof(int) || ValueType == typeof(float)) { for (int i = 0; i < CurBatch * ValueSize / 4; i++) { Array.Reverse(Bytes, i * 4, 4); } } else if (ValueType == typeof(double)) { for (int i = 0; i < CurBatch * ValueSize / 8; i++) { Array.Reverse(Bytes, i * 8, 8); } } } float *DataP = DataPtr + b; if (ValueType == typeof(byte)) { byte *BytesP = BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(short)) { short *BytesP = (short *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(ushort)) { ushort *BytesP = (ushort *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(int)) { int *BytesP = (int *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } else if (ValueType == typeof(float)) { float *BytesP = (float *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = *BytesP++; } } else if (ValueType == typeof(double)) { double *BytesP = (double *)BytesPtr; for (int i = 0; i < CurBatch; i++) { *DataP++ = (float)*BytesP++; } } } } } } } } else { Header = MapHeader.ReadFromFile(null, path, headerlessSliceDims, headerlessOffset, headerlessType, stream); if (Helper.PathToExtension(path).ToLower() == ".eer") { Data = Helper.ArrayOfFunction(i => new float[Header.Dimensions.ElementsSlice()], layers == null ? Header.Dimensions.Z : layers.Length); for (int i = 0; i < Data.Length; i++) { int z = layers == null ? i : layers[i]; EERNative.ReadEER(path, z * HeaderEER.GroupNFrames, (z + 1) * HeaderEER.GroupNFrames, HeaderEER.SuperResolution, Data[i]); } } else { Data = ((HeaderTiff)Header).ReadData(stream, layers); } } return(Data); }