private async void PerformRefinementIteration(ProcessingOptionsMPARefine options)
{
bool DoMultiProcess = true;
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount()];
string[] WorkerFolders = new string[Workers.Length];
string[] WorkerLogs = new string[Workers.Length];
foreach (var gpuID in UsedDevices)
{
WorkerFolders[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}");
Directory.CreateDirectory(WorkerFolders[gpuID]);
}
if (DoMultiProcess)
{
foreach (var gpuID in UsedDevices)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(2),
0,
"float");
WorkerLogs[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}", "run.out");
}
}
#endregion
var Progress = await this.ShowProgressAsync("Preparing for refinement – this will take a few minutes per species", "");
Progress.SetIndeterminate();
int ItemsCompleted = 0;
int ItemsToDo = ActivePopulation.Sources.Select(s => s.Files.Count).Sum();
long PinnedMemoryLimit = 1 << 30;
string[] CurrentlyRefinedItems = new string[GPU.GetDeviceCount()];
System.Timers.Timer StatusUpdater = null;
if (DoMultiProcess)
{
StatusUpdater = new System.Timers.Timer(1001);
StatusUpdater.Elapsed += (s, e) =>
{
lock (CurrentlyRefinedItems)
{
StringBuilder StatusMessage = new StringBuilder();
foreach (var gpuID in UsedDevices)
{
if (CurrentlyRefinedItems[gpuID] == null)
{
continue;
}
try
{
string ItemMessage = File.ReadLines(WorkerLogs[gpuID]).Last();
StatusMessage.Append(CurrentlyRefinedItems[gpuID] + ": " + ItemMessage + "\n");
}
catch { }
}
Dispatcher.Invoke(() => Progress.SetMessage(StatusMessage.ToString()));
}
};
}
try
{
await Task.Run(() =>
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Figuring out memory capacity..."));
WorkerWrapper[] MemoryTesters = Helper.ArrayOfFunction(i => new WorkerWrapper(0), 4);
try
{
int Tested = 0;
while (true)
{
long ChunkSize = (long)1 << 30; // 1 GB
long IncreaseBy = (long)1 << 31; // 2 GB
MemoryTesters[Tested % MemoryTesters.Length].TryAllocatePinnedMemory(Helper.ArrayOfConstant(ChunkSize, (int)(IncreaseBy / ChunkSize)));
PinnedMemoryLimit += IncreaseBy;
Tested++;
}
}
catch
{
PinnedMemoryLimit = PinnedMemoryLimit * 15 / 100; // Take 15% of that limit because Windows is weird
}
foreach (var item in MemoryTesters)
{
item.Dispose();
}
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preparing refinement requisites..."));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPAPreparePopulation(ActivePopulation.Path);
}, UsedDevices);
foreach (var species in ActivePopulation.Species)
{
species.PrepareRefinementRequisites(true, 0);
}
}
else
{
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preprocessing {species.Name}..."));
species.PrepareRefinementRequisites();
}
}
GPU.CheckGPUExceptions();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Performing refinement"));
Image.PrintObjectIDs();
if (true)
{
foreach (var source in ActivePopulation.Sources)
{
//break;
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Loading gain reference for {source.Name}..."));
Image[] GainRefs = new Image[GPU.GetDeviceCount()];
try
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
if (!string.IsNullOrEmpty(source.GainPath))
{
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose);
}
else
{
Workers[gpuID].LoadGainRef("", false, false, false);
}
}, UsedDevices);
}
else
{
Image GainRef = source.LoadAndPrepareGainReference();
if (GainRef != null)
{
GainRefs = Helper.ArrayOfFunction(i => GainRef.GetCopy(), GPU.GetDeviceCount());
}
}
}
catch
{
throw new Exception($"Could not load gain reference for {source.Name}.");
}
if (DoMultiProcess)
{
StatusUpdater.Start();
}
#region Load all items and determine pinned memory footprint for each of them
List <Movie> AllItems = source.Files.Select(pair => source.IsTiltSeries ? new TiltSeries(source.FolderPath + pair.Value) :
new Movie(source.FolderPath + pair.Value)).ToList();
Dictionary <Movie, long> ItemFootprints = new Dictionary <Movie, long>();
foreach (var item in AllItems)
{
ItemFootprints.Add(item, item.MultiParticleRefinementCalculateHostMemory(options, ActivePopulation.Species.ToArray(), source));
}
AllItems.Sort((a, b) => ItemFootprints[a].CompareTo(ItemFootprints[b]));
#endregion
long OverallFootprint = 0;
Queue <DeviceToken> Devices = new Queue <DeviceToken>();
for (int d = UsedDevices.Count - 1; d >= 0; d--)
{
Devices.Enqueue(new DeviceToken(UsedDevices[d]));
}
int NTokens = Devices.Count;
bool IsCanceled = false;
// A modified version of Helper.ForEachGPU()
int NDone = 0;
while (AllItems.Count > 0)
{
if (IsCanceled)
{
break;
}
//if (NDone++ < 200)
//{
// AllItems.RemoveAt(AllItems.Count - 1);
// continue;
//}
//if (NDone++ > 20)
// break;
while (Devices.Count <= 0)
{
Thread.Sleep(5);
}
DeviceToken CurrentDevice = null;
Movie CurrentItem = null;
while (CurrentItem == null)
{
int ItemID = AllItems.Count - 1;
lock (Devices) // Don't want OverallFootprint to change while checking
while (ItemID >= 0 && OverallFootprint + ItemFootprints[AllItems[ItemID]] > PinnedMemoryLimit)
{
ItemID--;
}
// No suitable item found and there is hope more memory will become available later
if (ItemID < 0 && OverallFootprint > 0)
{
Thread.Sleep(5);
continue;
}
// Either item can fit, or there is no hope for more memory later, so try anyway
if (ItemID < 0 && OverallFootprint == 0)
{
ItemID = AllItems.Count - 1;
}
ItemID = Math.Max(0, ItemID);
CurrentItem = AllItems[ItemID];
AllItems.Remove(CurrentItem);
lock (Devices)
{
CurrentDevice = Devices.Dequeue();
OverallFootprint += ItemFootprints[CurrentItem];
}
break;
}
Thread DeviceThread = new Thread(() =>
{
int GPUID = CurrentDevice.ID;
GPU.SetDevice(GPUID);
if (DoMultiProcess)
{
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = CurrentItem.Name;
Workers[GPUID].MPARefine(CurrentItem.Path,
WorkerFolders[GPUID],
WorkerLogs[GPUID],
options,
source);
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = null;
}
else
{
Dispatcher.InvokeAsync(() => Progress.SetTitle($"Refining {CurrentItem.Name}..."));
CurrentItem.PerformMultiParticleRefinement(WorkerFolders[GPUID], options, ActivePopulation.Species.ToArray(), source, GainRefs[GPUID], (s) =>
{
Dispatcher.InvokeAsync(() =>
{
Progress.SetMessage(s);
});
});
CurrentItem.SaveMeta();
GPU.CheckGPUExceptions();
}
Dispatcher.Invoke(() =>
{
ItemsCompleted++;
Progress.Maximum = ItemsToDo;
Progress.SetProgress(ItemsCompleted);
});
lock (Devices)
{
Devices.Enqueue(CurrentDevice);
OverallFootprint -= ItemFootprints[CurrentItem];
}
})
{
Name = $"ForEachGPU Device {CurrentDevice.ID}"
};
DeviceThread.Start();
}
while (Devices.Count != NTokens)
{
Thread.Sleep(5);
}
if (DoMultiProcess)
{
StatusUpdater.Stop();
}
source.Commit();
}
}
Image.PrintObjectIDs();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Finishing refinement"));
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage("Saving intermediate results"));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPASaveProgress(WorkerFolders[gpuID]);
Workers[gpuID].Dispose();
}, UsedDevices);
Dispatcher.InvokeAsync(() => Progress.SetMessage("Gathering intermediate results"));
ActivePopulation.GatherRefinementProgress(UsedDevices.Select(gpuID => WorkerFolders[gpuID]).ToArray());
foreach (var folder in WorkerFolders)
{
try
{
Directory.Delete(folder, true);
}
catch { }
}
}
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Reconstructing and filtering {species.Name}..."));
species.FinishRefinement();
species.Commit();
}
ActivePopulation.Save();
});
}
catch (Exception exc)
{
await Progress.CloseAsync();
await this.ShowMessageAsync("Oopsie", "Something went wrong during refinement. Sorry! Here are the details:\n\n" +
exc.ToString());
}
await Progress.CloseAsync();
}
private async void ButtonExport_OnClick(object sender, RoutedEventArgs e)
{
System.Windows.Forms.SaveFileDialog SaveDialog = new System.Windows.Forms.SaveFileDialog
{
Filter = "STAR Files|*.star"
};
System.Windows.Forms.DialogResult ResultSave = SaveDialog.ShowDialog();
if (ResultSave.ToString() == "OK")
{
ExportPath = SaveDialog.FileName;
}
else
{
return;
}
bool DoAverage = (bool)RadioAverage.IsChecked;
bool DoStack = (bool)RadioStack.IsChecked;
bool DoDenoisingPairs = (bool)RadioDenoising.IsChecked;
bool DoOnlyStar = (bool)RadioStar.IsChecked;
bool Invert = (bool)CheckInvert.IsChecked;
bool Normalize = (bool)CheckNormalize.IsChecked;
bool Preflip = (bool)CheckPreflip.IsChecked;
float AngPix = (float)Options.Tasks.InputPixelSize;
bool Relative = (bool)CheckRelative.IsChecked;
bool Filter = (bool)CheckFilter.IsChecked;
bool Manual = (bool)CheckManual.IsChecked;
int BoxSize = (int)Options.Tasks.Export2DBoxSize;
int NormDiameter = (int)Options.Tasks.Export2DParticleDiameter;
ProgressWrite.Visibility = Visibility.Visible;
ProgressWrite.IsIndeterminate = true;
PanelButtons.Visibility = Visibility.Collapsed;
PanelRemaining.Visibility = Visibility.Visible;
foreach (var element in DisableWhileProcessing)
{
element.IsEnabled = false;
}
await Task.Run(async() =>
{
#region Get all movies that can potentially be used
List <Movie> ValidMovies = Movies.Where(v =>
{
if (!Filter && v.UnselectFilter && v.UnselectManual == null)
{
return(false);
}
if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual)
{
return(false);
}
if (v.OptionsCTF == null)
{
return(false);
}
return(true);
}).ToList();
List <string> ValidMovieNames = ValidMovies.Select(m => m.RootName).ToList();
if (ValidMovies.Count == 0)
{
await Dispatcher.Invoke(async() =>
{
await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Oopsie",
"No items were found to extract particles from.\n" +
"Please make sure the names match, estimate the CTF for all items, and review your filtering thresholds.");
});
}
#endregion
#region Read table and intersect its micrograph set with valid movies
Star TableIn;
if (Options.Tasks.InputOnePerItem)
{
List <Star> Tables = new List <Star>();
foreach (var item in Movies)
{
string StarPath = InputFolder + item.RootName + InputSuffix + ".star";
if (File.Exists(StarPath))
{
Star TableItem = new Star(StarPath);
if (!TableItem.HasColumn("rlnMicrographName"))
{
TableItem.AddColumn("rlnMicrographName", item.Name);
}
if (item.PickingThresholds.ContainsKey(InputSuffix) && TableItem.HasColumn("rlnAutopickFigureOfMerit"))
{
float Threshold = (float)item.PickingThresholds[InputSuffix];
float[] Scores = TableItem.GetColumn("rlnAutopickFigureOfMerit").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
int[] InvalidRows = Helper.ArrayOfSequence(0, TableItem.RowCount, 1).Where(i => Scores[i] < Threshold).ToArray();
TableItem.RemoveRows(InvalidRows);
}
Tables.Add(TableItem);
}
}
TableIn = new Star(Tables.ToArray());
}
else
{
TableIn = new Star(ImportPath);
}
if (!TableIn.HasColumn("rlnMicrographName"))
{
throw new Exception("Couldn't find rlnMicrographName column.");
}
if (!TableIn.HasColumn("rlnCoordinateX"))
{
throw new Exception("Couldn't find rlnCoordinateX column.");
}
if (!TableIn.HasColumn("rlnCoordinateY"))
{
throw new Exception("Couldn't find rlnCoordinateY column.");
}
Dictionary <string, List <int> > Groups = new Dictionary <string, List <int> >();
{
string[] ColumnMicNames = TableIn.GetColumn("rlnMicrographName");
for (int r = 0; r < ColumnMicNames.Length; r++)
{
if (!Groups.ContainsKey(ColumnMicNames[r]))
{
Groups.Add(ColumnMicNames[r], new List <int>());
}
Groups[ColumnMicNames[r]].Add(r);
}
Groups = Groups.ToDictionary(group => Helper.PathToName(group.Key), group => group.Value);
Groups = Groups.Where(group => ValidMovieNames.Contains(group.Key)).ToDictionary(group => group.Key, group => group.Value);
}
bool[] RowsIncluded = new bool[TableIn.RowCount];
foreach (var group in Groups)
{
foreach (var r in group.Value)
{
RowsIncluded[r] = true;
}
}
List <int> RowsNotIncluded = new List <int>();
for (int r = 0; r < RowsIncluded.Length; r++)
{
if (!RowsIncluded[r])
{
RowsNotIncluded.Add(r);
}
}
ValidMovies = ValidMovies.Where(v => Groups.ContainsKey(v.RootName)).ToList();
if (ValidMovies.Count == 0) // Exit if there is nothing to export, otherwise errors will be thrown below
{
return;
}
#endregion
#region Make sure all columns are there
if (!TableIn.HasColumn("rlnMagnification"))
{
TableIn.AddColumn("rlnMagnification", "10000.0");
}
else
{
TableIn.SetColumn("rlnMagnification", Helper.ArrayOfConstant("10000.0", TableIn.RowCount));
}
if (!TableIn.HasColumn("rlnDetectorPixelSize"))
{
TableIn.AddColumn("rlnDetectorPixelSize", Options.GetProcessingParticleExport().BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture));
}
else
{
TableIn.SetColumn("rlnDetectorPixelSize", Helper.ArrayOfConstant(Options.GetProcessingParticleExport().BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture), TableIn.RowCount));
}
if (!TableIn.HasColumn("rlnVoltage"))
{
TableIn.AddColumn("rlnVoltage", "300.0");
}
if (!TableIn.HasColumn("rlnSphericalAberration"))
{
TableIn.AddColumn("rlnSphericalAberration", "2.7");
}
if (!TableIn.HasColumn("rlnAmplitudeContrast"))
{
TableIn.AddColumn("rlnAmplitudeContrast", "0.07");
}
if (!TableIn.HasColumn("rlnPhaseShift"))
{
TableIn.AddColumn("rlnPhaseShift", "0.0");
}
if (!TableIn.HasColumn("rlnDefocusU"))
{
TableIn.AddColumn("rlnDefocusU", "0.0");
}
if (!TableIn.HasColumn("rlnDefocusV"))
{
TableIn.AddColumn("rlnDefocusV", "0.0");
}
if (!TableIn.HasColumn("rlnDefocusAngle"))
{
TableIn.AddColumn("rlnDefocusAngle", "0.0");
}
if (!TableIn.HasColumn("rlnCtfMaxResolution"))
{
TableIn.AddColumn("rlnCtfMaxResolution", "999.0");
}
if (!TableIn.HasColumn("rlnImageName"))
{
TableIn.AddColumn("rlnImageName", "None");
}
if (!TableIn.HasColumn("rlnMicrographName"))
{
TableIn.AddColumn("rlnMicrographName", "None");
}
#endregion
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
List <int> UsedDeviceProcesses = Helper.Combine(Helper.ArrayOfFunction(i => UsedDevices.Select(d => d + i *NDevices).ToArray(), MainWindow.GlobalOptions.ProcessesPerDevice)).ToList();
if (IsCanceled)
{
return;
}
#region Create worker processes
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount() * MainWindow.GlobalOptions.ProcessesPerDevice];
foreach (var gpuID in UsedDeviceProcesses)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(Options.Import.HeaderlessWidth, Options.Import.HeaderlessHeight),
Options.Import.HeaderlessOffset,
Options.Import.HeaderlessType);
if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain)
{
Workers[gpuID].LoadGainRef(Options.Import.CorrectGain ? Options.Import.GainPath : "",
Options.Import.GainFlipX,
Options.Import.GainFlipY,
Options.Import.GainTranspose,
Options.Import.CorrectDefects ? Options.Import.DefectsPath : "");
}
}
#endregion
#region Load gain reference if needed
//Image[] ImageGain = new Image[NDevices];
//if (!string.IsNullOrEmpty(Options.Import.GainPath) && Options.Import.CorrectGain && File.Exists(Options.Import.GainPath))
// for (int d = 0; d < NDevices; d++)
// {
// GPU.SetDevice(d);
// ImageGain[d] = MainWindow.LoadAndPrepareGainReference();
// }
#endregion
bool Overwrite = true;
if (DoAverage || DoStack)
{
foreach (var movie in ValidMovies)
{
bool FileExists = File.Exists((DoAverage ? movie.ParticlesDir : movie.ParticleMoviesDir) + movie.RootName + Options.Tasks.OutputSuffix + ".mrcs");
if (FileExists)
{
await Dispatcher.Invoke(async() =>
{
var DialogResult = await((MainWindow)Application.Current.MainWindow).ShowMessageAsync("Some particle files already exist. Overwrite them?",
"",
MessageDialogStyle.AffirmativeAndNegative,
new MetroDialogSettings()
{
AffirmativeButtonText = "Yes",
NegativeButtonText = "No"
});
if (DialogResult == MessageDialogResult.Negative)
{
Overwrite = false;
}
});
break;
}
}
}
Star TableOut = null;
{
Dictionary <string, Star> MicrographTables = new Dictionary <string, Star>();
#region Get coordinates
float[] PosX = TableIn.GetColumn("rlnCoordinateX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] PosY = TableIn.GetColumn("rlnCoordinateY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] ShiftX = TableIn.HasColumn("rlnOriginX") ? TableIn.GetColumn("rlnOriginX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
float[] ShiftY = TableIn.HasColumn("rlnOriginY") ? TableIn.GetColumn("rlnOriginY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
for (int r = 0; r < TableIn.RowCount; r++)
{
PosX[r] -= ShiftX[r];
PosY[r] -= ShiftY[r];
}
if (TableIn.HasColumn("rlnOriginX"))
{
TableIn.RemoveColumn("rlnOriginX");
}
if (TableIn.HasColumn("rlnOriginY"))
{
TableIn.RemoveColumn("rlnOriginY");
}
#endregion
Dispatcher.Invoke(() => ProgressWrite.MaxValue = ValidMovies.Count);
Helper.ForEachGPU(ValidMovies, (movie, gpuID) =>
{
if (IsCanceled)
{
return;
}
Stopwatch ItemTime = new Stopwatch();
ItemTime.Start();
MapHeader OriginalHeader = MapHeader.ReadFromFile(movie.Path);
#region Set up export options
ProcessingOptionsParticlesExport ExportOptions = Options.GetProcessingParticleExport();
ExportOptions.DoAverage = DoAverage;
ExportOptions.DoDenoisingPairs = DoDenoisingPairs;
ExportOptions.DoStack = DoStack;
ExportOptions.Invert = Invert;
ExportOptions.Normalize = Normalize;
ExportOptions.PreflipPhases = Preflip;
ExportOptions.Dimensions = OriginalHeader.Dimensions.MultXY((float)Options.PixelSizeMean);
ExportOptions.BoxSize = BoxSize;
ExportOptions.Diameter = NormDiameter;
#endregion
bool FileExists = File.Exists((DoAverage ? movie.ParticlesDir : movie.ParticleMoviesDir) + movie.RootName + ExportOptions.Suffix + ".mrcs");
#region Load and prepare original movie
//Image OriginalStack = null;
decimal ScaleFactor = 1M / (decimal)Math.Pow(2, (double)ExportOptions.BinTimes);
if (!DoOnlyStar && (!FileExists || Overwrite))
{
Workers[gpuID].LoadStack(movie.Path, ScaleFactor, ExportOptions.EERGroupFrames);
}
//MainWindow.LoadAndPrepareHeaderAndMap(movie.Path, ImageGain[gpuID], ScaleFactor, out OriginalHeader, out OriginalStack);
if (IsCanceled)
{
//foreach (Image gain in ImageGain)
// gain?.Dispose();
//OriginalStack?.Dispose();
return;
}
#endregion
#region Figure out relative or absolute path to particle stack
string PathStack = (ExportOptions.DoStack ? movie.ParticleMoviesDir : movie.ParticlesDir) + movie.RootName + ExportOptions.Suffix + ".mrcs";
string PathMicrograph = movie.Path;
if (Relative)
{
Uri UriStar = new Uri(ExportPath);
PathStack = UriStar.MakeRelativeUri(new Uri(PathStack)).ToString();
PathMicrograph = UriStar.MakeRelativeUri(new Uri(PathMicrograph)).ToString();
}
#endregion
#region Update row values
List <int> GroupRows = Groups[movie.RootName];
List <float2> Positions = new List <float2>();
float Astigmatism = (float)movie.CTF.DefocusDelta / 2;
float PhaseShift = movie.OptionsCTF.DoPhase ? movie.GridCTFPhase.GetInterpolated(new float3(0.5f)) * 180 : 0;
int ImageNameIndex = TableIn.GetColumnID("rlnImageName");
foreach (var r in GroupRows)
{
float3 Position = new float3(PosX[r] * AngPix / ExportOptions.Dimensions.X,
PosY[r] * AngPix / ExportOptions.Dimensions.Y,
0.5f);
float LocalDefocus = movie.GridCTFDefocus.GetInterpolated(Position);
TableIn.SetRowValue(r, "rlnDefocusU", ((LocalDefocus + Astigmatism) * 1e4f).ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnDefocusV", ((LocalDefocus - Astigmatism) * 1e4f).ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnDefocusAngle", movie.CTF.DefocusAngle.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnVoltage", movie.CTF.Voltage.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnSphericalAberration", movie.CTF.Cs.ToString("F4", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnAmplitudeContrast", movie.CTF.Amplitude.ToString("F3", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnPhaseShift", PhaseShift.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCtfMaxResolution", movie.CTFResolutionEstimate.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateX", (PosX[r] * AngPix / (float)ExportOptions.BinnedPixelSizeMean).ToString("F2", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateY", (PosY[r] * AngPix / (float)ExportOptions.BinnedPixelSizeMean).ToString("F2", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnImageName", PathStack);
TableIn.SetRowValue(r, "rlnMicrographName", PathMicrograph);
Positions.Add(new float2(PosX[r] * AngPix,
PosY[r] * AngPix));
}
#endregion
#region Populate micrograph table with rows for all exported particles
Star MicrographTable = new Star(TableIn.GetColumnNames());
int StackDepth = (ExportOptions.DoAverage || ExportOptions.DoDenoisingPairs || DoOnlyStar)
? 1
: (OriginalHeader.Dimensions.Z - ExportOptions.SkipFirstN - ExportOptions.SkipLastN) /
ExportOptions.StackGroupSize;
int pi = 0;
for (int i = 0; i < StackDepth; i++)
{
foreach (var r in GroupRows)
{
List <string> Row = TableIn.GetRow(r).ToList();
Row[ImageNameIndex] = (++pi).ToString("D7") + "@" + Row[ImageNameIndex];
MicrographTable.AddRow(Row);
}
}
#endregion
#region Finally, process and export the actual particles
if (!DoOnlyStar && (!FileExists || Overwrite))
{
Workers[gpuID].MovieExportParticles(movie.Path, ExportOptions, Positions.ToArray());
//movie.ExportParticles(OriginalStack, Positions.ToArray(), ExportOptions);
//OriginalStack.Dispose();
}
#endregion
#region Add this micrograph's table to global collection, update remaining time estimate
lock (MicrographTables)
{
MicrographTables.Add(movie.RootName, MicrographTable);
Timings.Add(ItemTime.ElapsedMilliseconds / (float)NDevices);
int MsRemaining = (int)(MathHelper.Mean(Timings) * (ValidMovies.Count - MicrographTables.Count));
TimeSpan SpanRemaining = new TimeSpan(0, 0, 0, 0, MsRemaining);
Dispatcher.Invoke(() => TextRemaining.Text = SpanRemaining.ToString((int)SpanRemaining.TotalHours > 0 ? @"hh\:mm\:ss" : @"mm\:ss"));
Dispatcher.Invoke(() =>
{
ProgressWrite.IsIndeterminate = false;
ProgressWrite.Value = MicrographTables.Count;
});
}
#endregion
}, 1, UsedDeviceProcesses);
if (MicrographTables.Count > 0)
{
TableOut = new Star(MicrographTables.Values.ToArray());
}
}
Thread.Sleep(10000); // Writing out particle stacks is async, so if workers are killed immediately they may not write out everything
foreach (var worker in Workers)
{
worker?.Dispose();
}
//foreach (Image gain in ImageGain)
// gain?.Dispose();
if (IsCanceled)
{
return;
}
TableOut.Save(ExportPath);
});
DataContext = null;
Close?.Invoke();
}
private async void PerformRefinementIteration(ProcessingOptionsMPARefine options)
{
bool DoMultiProcess = true;
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount()];
string[] WorkerFolders = new string[Workers.Length];
string[] WorkerLogs = new string[Workers.Length];
foreach (var gpuID in UsedDevices)
{
WorkerFolders[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}");
Directory.CreateDirectory(WorkerFolders[gpuID]);
}
if (DoMultiProcess)
{
foreach (var gpuID in UsedDevices)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(2),
0,
"float");
WorkerLogs[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}", "run.out");
}
}
#endregion
var Progress = await this.ShowProgressAsync("Preparing for refinement – this will take a few minutes per species", "");
Progress.SetIndeterminate();
int ItemsCompleted = 0;
int ItemsToDo = ActivePopulation.Sources.Select(s => s.Files.Count).Sum();
string[] CurrentlyRefinedItems = new string[GPU.GetDeviceCount()];
System.Timers.Timer StatusUpdater = null;
if (DoMultiProcess)
{
StatusUpdater = new System.Timers.Timer(1001);
StatusUpdater.Elapsed += (s, e) =>
{
lock (CurrentlyRefinedItems)
{
StringBuilder StatusMessage = new StringBuilder();
foreach (var gpuID in UsedDevices)
{
if (CurrentlyRefinedItems[gpuID] == null)
{
continue;
}
try
{
string ItemMessage = File.ReadLines(WorkerLogs[gpuID]).Last();
StatusMessage.Append(CurrentlyRefinedItems[gpuID] + ": " + ItemMessage + "\n");
} catch { }
}
Dispatcher.Invoke(() => Progress.SetMessage(StatusMessage.ToString()));
}
};
}
try
{
await Task.Run(() =>
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPAPreparePopulation(ActivePopulation.Path);
}, UsedDevices);
foreach (var species in ActivePopulation.Species)
{
species.PrepareRefinementRequisites(false, 0);
}
}
else
{
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preprocessing {species.Name}..."));
species.PrepareRefinementRequisites();
}
}
GPU.CheckGPUExceptions();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Performing refinement"));
Image.PrintObjectIDs();
//if (false)
foreach (var source in ActivePopulation.Sources)
{
//break;
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Loading gain reference for {source.Name}..."));
Image[] GainRefs = new Image[GPU.GetDeviceCount()];
try
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
if (!string.IsNullOrEmpty(source.GainPath))
{
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose);
}
else
{
Workers[gpuID].LoadGainRef("", false, false, false);
}
}, UsedDevices);
}
else
{
Image GainRef = source.LoadAndPrepareGainReference();
if (GainRef != null)
{
GainRefs = Helper.ArrayOfFunction(i => GainRef.GetCopy(), GPU.GetDeviceCount());
}
}
}
catch
{
throw new Exception($"Could not load gain reference for {source.Name}.");
}
if (DoMultiProcess)
{
StatusUpdater.Start();
}
Helper.ForEachGPU(source.Files, (pair, GPUID) =>
{
if (DoMultiProcess)
{
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = pair.Value;
Workers[GPUID].MPARefine(source.FolderPath + pair.Value,
WorkerFolders[GPUID],
WorkerLogs[GPUID],
options,
source);
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = null;
}
else
{
Movie Item = null;
if (source.IsTiltSeries)
{
Item = new TiltSeries(source.FolderPath + pair.Value);
}
else
{
Item = new Movie(source.FolderPath + pair.Value);
}
Dispatcher.InvokeAsync(() => Progress.SetTitle($"Refining {Item.Name}..."));
Item.PerformMultiParticleRefinement(WorkerFolders[GPUID], options, ActivePopulation.Species.ToArray(), source, GainRefs[GPUID], (s) =>
{
Dispatcher.InvokeAsync(() =>
{
Progress.SetMessage(s);
});
});
Item.SaveMeta();
GPU.CheckGPUExceptions();
}
Dispatcher.Invoke(() =>
{
ItemsCompleted++;
Progress.Maximum = ItemsToDo;
Progress.SetProgress(ItemsCompleted);
});
return(false);
}, 1, UsedDevices);
if (DoMultiProcess)
{
StatusUpdater.Stop();
}
source.Commit();
}
Image.PrintObjectIDs();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Finishing refinement"));
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage("Saving intermediate results"));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPASaveProgress(WorkerFolders[gpuID]);
Workers[gpuID].Dispose();
}, UsedDevices);
Dispatcher.InvokeAsync(() => Progress.SetMessage("Gathering intermediate results"));
ActivePopulation.GatherRefinementProgress(UsedDevices.Select(gpuID => WorkerFolders[gpuID]).ToArray());
foreach (var folder in WorkerFolders)
{
try
{
Directory.Delete(folder, true);
}
catch { }
}
}
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Reconstructing and filtering {species.Name}..."));
species.FinishRefinement();
species.Commit();
}
});
}
catch (Exception exc)
{
await Progress.CloseAsync();
await this.ShowMessageAsync("Oopsie", "Something went wrong during refinement. Sorry! Here are the details:\n\n" +
exc.ToString());
}
await Progress.CloseAsync();
}
private async void ButtonWrite_OnClick(object sender, RoutedEventArgs e)
{
System.Windows.Forms.SaveFileDialog SaveDialog = new System.Windows.Forms.SaveFileDialog
{
Filter = "STAR Files|*.star"
};
System.Windows.Forms.DialogResult ResultSave = SaveDialog.ShowDialog();
if (ResultSave.ToString() == "OK")
{
ExportPath = SaveDialog.FileName;
}
else
{
return;
}
bool Invert = (bool)CheckInvert.IsChecked;
bool Normalize = (bool)CheckNormalize.IsChecked;
bool Preflip = (bool)CheckPreflip.IsChecked;
bool Relative = (bool)CheckRelative.IsChecked;
bool Filter = (bool)CheckFilter.IsChecked;
bool Manual = (bool)CheckManual.IsChecked;
int BoxSize = (int)Options.Tasks.Export2DBoxSize;
int NormDiameter = (int)Options.Tasks.Export2DParticleDiameter;
bool DoVolumes = (bool)RadioVolume.IsChecked;
if (!DoVolumes)
{
Options.Tasks.TomoSubReconstructPrerotated = true;
}
bool MakeSparse = (bool)CheckSparse.IsChecked;
float3 AdditionalShiftAngstrom = (bool)CheckShiftParticles.IsChecked ?
new float3((float)SliderShiftParticlesX.Value,
(float)SliderShiftParticlesY.Value,
(float)SliderShiftParticlesZ.Value) :
new float3(0);
ProgressWrite.Visibility = Visibility.Visible;
ProgressWrite.IsIndeterminate = true;
PanelButtons.Visibility = Visibility.Collapsed;
PanelRemaining.Visibility = Visibility.Visible;
foreach (var element in DisableWhileProcessing)
{
element.IsEnabled = false;
}
await Task.Run(() =>
{
#region Get all movies that can potentially be used
List <TiltSeries> ValidSeries = Series.Where(v =>
{
if (!Filter && v.UnselectFilter && v.UnselectManual == null)
{
return(false);
}
if (!Manual && v.UnselectManual != null && (bool)v.UnselectManual)
{
return(false);
}
if (v.OptionsCTF == null)
{
return(false);
}
return(true);
}).ToList();
List <string> ValidMovieNames = ValidSeries.Select(m => m.RootName).ToList();
#endregion
#region Read table and intersect its micrograph set with valid movies
Star TableIn;
if (Options.Tasks.InputOnePerItem)
{
List <Star> Tables = new List <Star>();
foreach (var item in Series)
{
string StarPath = InputFolder + item.RootName + InputSuffix;
if (File.Exists(StarPath))
{
Star TableItem = new Star(StarPath);
if (!TableItem.HasColumn("rlnMicrographName"))
{
TableItem.AddColumn("rlnMicrographName", item.Name);
}
else
{
TableItem.SetColumn("rlnMicrographName", Helper.ArrayOfConstant(item.Name, TableItem.RowCount));
}
Tables.Add(TableItem);
}
}
TableIn = new Star(Tables.ToArray());
}
else
{
TableIn = new Star(ImportPath);
}
if (!TableIn.HasColumn("rlnMicrographName"))
{
throw new Exception("Couldn't find rlnMicrographName column.");
}
if (!TableIn.HasColumn("rlnCoordinateX"))
{
throw new Exception("Couldn't find rlnCoordinateX column.");
}
if (!TableIn.HasColumn("rlnCoordinateY"))
{
throw new Exception("Couldn't find rlnCoordinateY column.");
}
if (!TableIn.HasColumn("rlnCoordinateZ"))
{
throw new Exception("Couldn't find rlnCoordinateZ column.");
}
Dictionary <string, List <int> > Groups = new Dictionary <string, List <int> >();
{
string[] ColumnMicNames = TableIn.GetColumn("rlnMicrographName");
for (int r = 0; r < ColumnMicNames.Length; r++)
{
if (!Groups.ContainsKey(ColumnMicNames[r]))
{
Groups.Add(ColumnMicNames[r], new List <int>());
}
Groups[ColumnMicNames[r]].Add(r);
}
Groups = Groups.ToDictionary(group => Helper.PathToName(group.Key), group => group.Value);
Groups = Groups.Where(group => ValidMovieNames.Any(n => group.Key.Contains(n))).ToDictionary(group => group.Key, group => group.Value);
}
bool[] RowsIncluded = new bool[TableIn.RowCount];
foreach (var group in Groups)
{
foreach (var r in group.Value)
{
RowsIncluded[r] = true;
}
}
List <int> RowsNotIncluded = new List <int>();
for (int r = 0; r < RowsIncluded.Length; r++)
{
if (!RowsIncluded[r])
{
RowsNotIncluded.Add(r);
}
}
ValidSeries = ValidSeries.Where(v => Groups.Any(n => n.Key.Contains(v.RootName))).ToList();
if (ValidSeries.Count == 0) // Exit if there is nothing to export, otherwise errors will be thrown below
{
return;
}
#endregion
#region Make sure all columns are there
if (!TableIn.HasColumn("rlnMagnification"))
{
TableIn.AddColumn("rlnMagnification", "10000.0");
}
else
{
TableIn.SetColumn("rlnMagnification", Helper.ArrayOfConstant("10000.0", TableIn.RowCount));
}
if (!TableIn.HasColumn("rlnDetectorPixelSize"))
{
TableIn.AddColumn("rlnDetectorPixelSize", Options.Tasks.TomoSubReconstructPixel.ToString("F5", CultureInfo.InvariantCulture));
}
else
{
TableIn.SetColumn("rlnDetectorPixelSize", Helper.ArrayOfConstant(Options.Tasks.TomoSubReconstructPixel.ToString("F5", CultureInfo.InvariantCulture), TableIn.RowCount));
}
if (!TableIn.HasColumn("rlnCtfMaxResolution"))
{
TableIn.AddColumn("rlnCtfMaxResolution", "999.0");
}
if (!TableIn.HasColumn("rlnImageName"))
{
TableIn.AddColumn("rlnImageName", "None");
}
if (!TableIn.HasColumn("rlnCtfImage"))
{
TableIn.AddColumn("rlnCtfImage", "None");
}
List <Star> SeriesTablesOut = new List <Star>();
#endregion
if (IsCanceled)
{
return;
}
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
List <int> UsedDeviceProcesses = Helper.Combine(Helper.ArrayOfFunction(i => UsedDevices.Select(d => d + i *NDevices).ToArray(), MainWindow.GlobalOptions.ProcessesPerDevice)).ToList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount() * MainWindow.GlobalOptions.ProcessesPerDevice];
foreach (var gpuID in UsedDeviceProcesses)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(Options.Import.HeaderlessWidth, Options.Import.HeaderlessHeight),
Options.Import.HeaderlessOffset,
Options.Import.HeaderlessType);
Workers[gpuID].LoadGainRef(Options.Import.CorrectGain ? Options.Import.GainPath : "",
Options.Import.GainFlipX,
Options.Import.GainFlipY,
Options.Import.GainTranspose,
Options.Import.CorrectDefects ? Options.Import.DefectsPath : "");
}
#endregion
Star TableOut = null;
{
Dictionary <string, Star> MicrographTables = new Dictionary <string, Star>();
#region Get coordinates and angles
float[] PosX = TableIn.GetColumn("rlnCoordinateX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] PosY = TableIn.GetColumn("rlnCoordinateY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] PosZ = TableIn.GetColumn("rlnCoordinateZ").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray();
float[] ShiftX = TableIn.HasColumn("rlnOriginX") ? TableIn.GetColumn("rlnOriginX").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
float[] ShiftY = TableIn.HasColumn("rlnOriginY") ? TableIn.GetColumn("rlnOriginY").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
float[] ShiftZ = TableIn.HasColumn("rlnOriginZ") ? TableIn.GetColumn("rlnOriginZ").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
if (Options.Tasks.TomoSubReconstructNormalizedCoords)
{
for (int r = 0; r < TableIn.RowCount; r++)
{
PosX[r] *= (float)Options.Tomo.DimensionsX * (float)Options.PixelSizeMean;
PosY[r] *= (float)Options.Tomo.DimensionsY * (float)Options.PixelSizeMean;
PosZ[r] *= (float)Options.Tomo.DimensionsZ * (float)Options.PixelSizeMean;
}
}
else
{
for (int r = 0; r < TableIn.RowCount; r++)
{
PosX[r] = (PosX[r] - ShiftX[r]) * (float)Options.Tasks.InputPixelSize;
PosY[r] = (PosY[r] - ShiftY[r]) * (float)Options.Tasks.InputPixelSize;
PosZ[r] = (PosZ[r] - ShiftZ[r]) * (float)Options.Tasks.InputPixelSize;
}
}
float[] AngleRot = TableIn.HasColumn("rlnAngleRot") && Options.Tasks.TomoSubReconstructPrerotated ? TableIn.GetColumn("rlnAngleRot").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
float[] AngleTilt = TableIn.HasColumn("rlnAngleTilt") && Options.Tasks.TomoSubReconstructPrerotated ? TableIn.GetColumn("rlnAngleTilt").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
float[] AnglePsi = TableIn.HasColumn("rlnAnglePsi") && Options.Tasks.TomoSubReconstructPrerotated ? TableIn.GetColumn("rlnAnglePsi").Select(v => float.Parse(v, CultureInfo.InvariantCulture)).ToArray() : new float[TableIn.RowCount];
if (TableIn.HasColumn("rlnOriginX"))
{
TableIn.RemoveColumn("rlnOriginX");
}
if (TableIn.HasColumn("rlnOriginY"))
{
TableIn.RemoveColumn("rlnOriginY");
}
if (TableIn.HasColumn("rlnOriginZ"))
{
TableIn.RemoveColumn("rlnOriginZ");
}
if (AdditionalShiftAngstrom.Length() > 0)
{
for (int r = 0; r < TableIn.RowCount; r++)
{
Matrix3 R = Matrix3.Euler(AngleRot[r] * Helper.ToRad,
AngleTilt[r] * Helper.ToRad,
AnglePsi[r] * Helper.ToRad);
float3 RotatedShift = R *AdditionalShiftAngstrom;
PosX[r] += RotatedShift.X;
PosY[r] += RotatedShift.Y;
PosZ[r] += RotatedShift.Z;
}
}
if (Options.Tasks.TomoSubReconstructPrerotated)
{
if (TableIn.HasColumn("rlnAngleRot"))
{
TableIn.RemoveColumn("rlnAngleRot");
TableIn.AddColumn("rlnAngleRot", "0");
}
if (TableIn.HasColumn("rlnAngleTilt"))
{
TableIn.RemoveColumn("rlnAngleTilt");
TableIn.AddColumn("rlnAngleTilt", "0");
}
if (TableIn.HasColumn("rlnAnglePsi"))
{
TableIn.RemoveColumn("rlnAnglePsi");
TableIn.AddColumn("rlnAnglePsi", "0");
}
}
#endregion
Dispatcher.Invoke(() => ProgressWrite.MaxValue = ValidSeries.Count);
//Dispatcher.Invoke(() =>
//{
// ProgressWrite.IsIndeterminate = true;
// TextRemaining.Text = "?:??";
//});
Helper.ForEachGPU(ValidSeries, (series, gpuID) =>
{
if (IsCanceled)
{
return;
}
Stopwatch ItemTime = new Stopwatch();
ItemTime.Start();
ProcessingOptionsTomoSubReconstruction ExportOptions = Options.GetProcessingTomoSubReconstruction();
#region Update row values
List <int> GroupRows = Groups.First(n => n.Key.Contains(series.RootName)).Value;
int pi = 0;
foreach (var r in GroupRows)
{
TableIn.SetRowValue(r, "rlnCtfMaxResolution", series.CTFResolutionEstimate.ToString("F1", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateX", (PosX[r] / (float)ExportOptions.BinnedPixelSizeMean).ToString("F3", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateY", (PosY[r] / (float)ExportOptions.BinnedPixelSizeMean).ToString("F3", CultureInfo.InvariantCulture));
TableIn.SetRowValue(r, "rlnCoordinateZ", (PosZ[r] / (float)ExportOptions.BinnedPixelSizeMean).ToString("F3", CultureInfo.InvariantCulture));
#region Figure out relative or absolute path to sub-tomo and its CTF
string PathSubtomo = series.SubtomoDir + $"{series.RootName}{ExportOptions.Suffix}_{pi:D7}_{ExportOptions.BinnedPixelSizeMean:F2}A.mrc";
string PathCTF = //MakeSparse ?
//(series.SubtomoDir + $"{series.RootName}_{pi:D7}_ctf_{ExportOptions.BinnedPixelSizeMean:F2}A.tif") :
(series.SubtomoDir + $"{series.RootName}{ExportOptions.Suffix}_{pi:D7}_ctf_{ExportOptions.BinnedPixelSizeMean:F2}A.mrc");
if (Relative)
{
Uri UriStar = new Uri(ExportPath);
PathSubtomo = UriStar.MakeRelativeUri(new Uri(PathSubtomo)).ToString();
PathCTF = UriStar.MakeRelativeUri(new Uri(PathCTF)).ToString();
}
#endregion
TableIn.SetRowValue(r, "rlnImageName", PathSubtomo);
TableIn.SetRowValue(r, "rlnCtfImage", PathCTF);
pi++;
}
#endregion
#region Populate micrograph table with rows for all exported particles
Star MicrographTable = new Star(TableIn.GetColumnNames());
foreach (var r in GroupRows)
{
MicrographTable.AddRow(TableIn.GetRow(r).ToList());
}
#endregion
#region Finally, reconstruct the actual sub-tomos
float3[] TomoPositions = Helper.Combine(GroupRows.Select(r => Helper.ArrayOfConstant(new float3(PosX[r], PosY[r], PosZ[r]), series.NTilts)).ToArray());
float3[] TomoAngles = Helper.Combine(GroupRows.Select(r => Helper.ArrayOfConstant(new float3(AngleRot[r], AngleTilt[r], AnglePsi[r]), series.NTilts)).ToArray());
if (DoVolumes)
{
Workers[gpuID].TomoExportParticles(series.Path, ExportOptions, TomoPositions, TomoAngles);
//series.ReconstructSubtomos(ExportOptions, TomoPositions, TomoAngles);
lock (MicrographTables)
MicrographTables.Add(series.RootName, MicrographTable);
}
else
{
Star SeriesTable;
Random Rand = new Random(123);
int[] Subsets = Helper.ArrayOfFunction(i => Rand.Next(1, 3), GroupRows.Count);
series.ReconstructParticleSeries(ExportOptions, TomoPositions, TomoAngles, Subsets, ExportPath, out SeriesTable);
lock (MicrographTables)
MicrographTables.Add(series.RootName, SeriesTable);
}
#endregion
#region Add this micrograph's table to global collection, update remaining time estimate
lock (MicrographTables)
{
Timings.Add(ItemTime.ElapsedMilliseconds / (float)UsedDeviceProcesses.Count);
int MsRemaining = (int)(MathHelper.Mean(Timings) * (ValidSeries.Count - MicrographTables.Count));
TimeSpan SpanRemaining = new TimeSpan(0, 0, 0, 0, MsRemaining);
Dispatcher.Invoke(() => TextRemaining.Text = SpanRemaining.ToString((int)SpanRemaining.TotalHours > 0 ? @"hh\:mm\:ss" : @"mm\:ss"));
Dispatcher.Invoke(() =>
{
ProgressWrite.IsIndeterminate = false;
ProgressWrite.Value = MicrographTables.Count;
});
}
#endregion
}, 1, UsedDeviceProcesses);
if (MicrographTables.Count > 0)
{
TableOut = new Star(MicrographTables.Values.ToArray());
}
}
Thread.Sleep(10000); // Writing out particles is async, so if workers are killed immediately they may not write out everything
foreach (var worker in Workers)
{
worker?.Dispose();
}
if (IsCanceled)
{
return;
}
TableOut.Save(ExportPath);
});
Close?.Invoke();
}
private async void ButtonWrite_OnClick(object sender, RoutedEventArgs e)
{
System.Windows.Forms.SaveFileDialog SaveDialog = new System.Windows.Forms.SaveFileDialog
{
Filter = "STAR Files|*.star"
};
System.Windows.Forms.DialogResult ResultSave = SaveDialog.ShowDialog();
if (ResultSave.ToString() == "OK")
{
ExportPath = SaveDialog.FileName;
}
else
{
return;
}
decimal _OutputAngPix = OutputAngPix;
int _BoxSize = BoxSize;
int _Diameter = Diameter;
bool _ReconstructVolume = ReconstructVolume;
bool _ReconstructSeries = ReconstructSeries;
bool _ApplyShift = ApplyShift;
decimal _ShiftX = ShiftX;
decimal _ShiftY = ShiftY;
decimal _ShiftZ = ShiftZ;
bool _ReconstructPrerotated = ReconstructPrerotated;
bool _ReconstructDoLimitDose = ReconstructDoLimitDose;
int _ReconstructNTilts = ReconstructNTilts;
bool _InputInvert = InputInvert;
bool _InputNormalize = InputNormalize;
bool _OutputNormalize = OutputNormalize;
bool _ReconstructMakeSparse = ReconstructMakeSparse;
if (!_ReconstructVolume)
{
_ReconstructPrerotated = true;
}
float3 AdditionalShiftAngstrom = new float3((float)_ShiftX,
(float)_ShiftY,
(float)_ShiftZ);
ProgressWrite.Visibility = Visibility.Visible;
ProgressWrite.IsIndeterminate = true;
PanelButtons.Visibility = Visibility.Collapsed;
PanelRemaining.Visibility = Visibility.Visible;
foreach (var element in DisableWhileProcessing)
{
element.IsEnabled = false;
}
MainWindow MainWindow = (MainWindow)Application.Current.MainWindow;
await Task.Run(() =>
{
List <Star> AllSourceTables = new List <Star>();
string[] AllSourceHashes = Helper.GetUniqueElements(Species.Particles.Select(p => p.SourceHash)).ToArray();
AllSourceHashes = AllSourceHashes.Where(h => Sources.Any(s => s.Files.ContainsKey(h))).ToArray();
foreach (var source in Sources)
{
if (!source.IsTiltSeries)
{
continue;
}
#region Get all movies that can potentially be used
List <string> ValidSourceHashes = AllSourceHashes.Where(h => source.Files.ContainsKey(h)).ToList();
List <string> ValidSourcePaths = ValidSourceHashes.Select(k => System.IO.Path.Combine(source.FolderPath, source.Files[k])).ToList();
List <string> ValidMovieNames = ValidSourcePaths.Select(p => Helper.PathToName(p)).ToList();
List <TiltSeries> ValidSeries = ValidSourcePaths.Select(p => new TiltSeries(p)).ToList();
List <Particle> Particles = Species.Particles.Where(p => ValidSourceHashes.Contains(p.SourceHash)).ToList();
Particles.Sort((a, b) => a.SourceHash.CompareTo(b.SourceHash));
#endregion
if (IsCanceled)
{
return;
}
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = MainWindow.GetDeviceList();
List <int> UsedDeviceProcesses = Helper.Combine(Helper.ArrayOfFunction(i => UsedDevices.Select(d => d + i *NDevices).ToArray(), 1)).ToList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount() * 1];
foreach (var gpuID in UsedDeviceProcesses)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(1, 1),
0,
"float32");
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose,
source.DefectsPath);
}
#endregion
List <Star> SourceTables = new List <Star>();
{
Dispatcher.Invoke(() => ProgressWrite.MaxValue = AllSourceHashes.Length);
Helper.ForEachGPU(ValidSeries, (series, gpuID) =>
{
if (IsCanceled)
{
return;
}
Stopwatch ItemTime = new Stopwatch();
ItemTime.Start();
string SeriesHash = series.GetDataHash();
Star TableOut = new Star(new string[] { "rlnMagnification",
"rlnDetectorPixelSize",
"rlnCoordinateX",
"rlnCoordinateY",
"rlnCoordinateZ",
"rlnAngleRot",
"rlnAngleTilt",
"rlnAnglePsi",
"rlnImageName",
"rlnCtfImage",
"rlnRandomSubset" });
ProcessingOptionsTomoSubReconstruction ExportOptions = new ProcessingOptionsTomoSubReconstruction()
{
PixelSizeX = source.PixelSizeX,
PixelSizeY = source.PixelSizeY,
PixelSizeAngle = source.PixelSizeAngle,
BinTimes = (decimal)Math.Log((double)(_OutputAngPix / source.PixelSizeMean), 2.0),
GainPath = source.GainPath,
DefectsPath = source.DefectsPath,
GainFlipX = source.GainFlipX,
GainFlipY = source.GainFlipY,
GainTranspose = source.GainTranspose,
Dimensions = new float3((float)source.DimensionsX,
(float)source.DimensionsY,
(float)source.DimensionsZ),
Suffix = "_" + Species.NameSafe,
BoxSize = _BoxSize,
ParticleDiameter = _Diameter,
Invert = _InputInvert,
NormalizeInput = _InputNormalize,
NormalizeOutput = _OutputNormalize,
PrerotateParticles = _ReconstructPrerotated,
DoLimitDose = _ReconstructDoLimitDose,
NTilts = Math.Min(series.NTilts, Math.Min(source.FrameLimit, _ReconstructNTilts)),
MakeSparse = _ReconstructMakeSparse
};
Particle[] SeriesParticles = Particles.Where(p => p.SourceHash == SeriesHash).ToArray();
int NParticles = SeriesParticles.Length;
#region Process particle positions and angles
float3[] Positions = new float3[NParticles *series.NTilts];
float3[] Angles = new float3[NParticles *series.NTilts];
int[] Subsets = SeriesParticles.Select(p => p.RandomSubset + 1).ToArray();
float MinDose = MathHelper.Min(series.Dose);
float MaxDose = MathHelper.Max(series.Dose);
float[] InterpolationSteps = Helper.ArrayOfFunction(i => (series.Dose[i] - MinDose) / (MaxDose - MinDose), series.NTilts);
for (int p = 0; p < NParticles; p++)
{
float3[] ParticlePositions = SeriesParticles[p].GetCoordinateSeries(InterpolationSteps);
float3[] ParticleAngles = SeriesParticles[p].GetAngleSeries(InterpolationSteps);
if (_ApplyShift)
{
Matrix3 R0 = Matrix3.Euler(ParticleAngles[0] * Helper.ToRad);
float3 RotatedShift = R0 *AdditionalShiftAngstrom;
for (int t = 0; t < ParticlePositions.Length; t++)
{
ParticlePositions[t] += RotatedShift;
}
}
if (!_ReconstructPrerotated)
{
Matrix3 R0I = Matrix3.Euler(ParticleAngles[0] * Helper.ToRad).Transposed();
for (int t = 0; t < ParticleAngles.Length; t++)
{
ParticleAngles[t] = Matrix3.EulerFromMatrix(R0I *Matrix3.Euler(ParticleAngles[t] * Helper.ToRad)) * Helper.ToDeg;
}
}
for (int t = 0; t < series.NTilts; t++)
{
Positions[p *series.NTilts + t] = ParticlePositions[t];
Angles[p *series.NTilts + t] = ParticleAngles[t];
}
string PathSubtomo = series.SubtomoDir + $"{series.RootName}{ExportOptions.Suffix}_{p:D7}_{ExportOptions.BinnedPixelSizeMean:F2}A.mrc";
string PathCTF = (series.SubtomoDir + $"{series.RootName}{ExportOptions.Suffix}_{p:D7}_ctf_{ExportOptions.BinnedPixelSizeMean:F2}A.mrc");
Uri UriStar = new Uri(ExportPath);
PathSubtomo = UriStar.MakeRelativeUri(new Uri(PathSubtomo)).ToString();
PathCTF = UriStar.MakeRelativeUri(new Uri(PathCTF)).ToString();
TableOut.AddRow(new List <string>()
{
"10000.0",
ExportOptions.BinnedPixelSizeMean.ToString("F5", CultureInfo.InvariantCulture),
(ParticlePositions[0].X / (float)ExportOptions.BinnedPixelSizeMean).ToString("F5", CultureInfo.InvariantCulture),
(ParticlePositions[0].Y / (float)ExportOptions.BinnedPixelSizeMean).ToString("F5", CultureInfo.InvariantCulture),
(ParticlePositions[0].Z / (float)ExportOptions.BinnedPixelSizeMean).ToString("F5", CultureInfo.InvariantCulture),
(_ReconstructPrerotated ? 0 : SeriesParticles[p].Angles[0].X).ToString("F5", CultureInfo.InvariantCulture),
(_ReconstructPrerotated ? 0 : SeriesParticles[p].Angles[0].Y).ToString("F5", CultureInfo.InvariantCulture),
(_ReconstructPrerotated ? 0 : SeriesParticles[p].Angles[0].Z).ToString("F5", CultureInfo.InvariantCulture),
PathSubtomo,
PathCTF,
Subsets[p].ToString()
});
}
#endregion
#region Finally, reconstruct the actual sub-tomos
if (_ReconstructVolume)
{
Workers[gpuID].TomoExportParticles(series.Path, ExportOptions, Positions, Angles);
}
else
{
series.ReconstructParticleSeries(ExportOptions, Positions, Angles, Subsets, ExportPath, out TableOut);
}
lock (AllSourceTables)
AllSourceTables.Add(TableOut);
#endregion
#region Add this micrograph's table to global collection, update remaining time estimate
lock (AllSourceTables)
{
Timings.Add(ItemTime.ElapsedMilliseconds / (float)UsedDeviceProcesses.Count);
int MsRemaining = (int)(MathHelper.Mean(Timings) * (AllSourceHashes.Length - AllSourceTables.Count));
TimeSpan SpanRemaining = new TimeSpan(0, 0, 0, 0, MsRemaining);
Dispatcher.Invoke(() => TextRemaining.Text = SpanRemaining.ToString((int)SpanRemaining.TotalHours > 0 ? @"hh\:mm\:ss" : @"mm\:ss"));
Dispatcher.Invoke(() =>
{
ProgressWrite.IsIndeterminate = false;
ProgressWrite.Value = AllSourceTables.Count;
});
}
#endregion
}, 1, UsedDeviceProcesses);
}
Thread.Sleep(10000); // Writing out particles is async, so if workers are killed immediately they may not write out everything
foreach (var worker in Workers)
{
worker?.Dispose();
}
if (IsCanceled)
{
return;
}
}
if (AllSourceTables.Count > 0)
{
(new Star(AllSourceTables.ToArray())).Save(ExportPath);
}
});
Close?.Invoke();
}
