public void MPARefine(string path, string workingDirectory, string logPath, ProcessingOptionsMPARefine options, DataSource source)
{
if (!SendCommand(new NamedSerializableObject("MPARefine",
path,
workingDirectory,
logPath,
options,
source)))
{
throw new Exception("Couldn't perform MPA refinement!");
}
}
private async void ButtonPopulationRefine_Click(object sender, RoutedEventArgs e)
{
if (ActivePopulation == null)
{
return;
}
//if (ActivePopulation.Species.Count > 1)
//{
// await this.ShowMessageAsync("Oopsie", "Multiple species aren't supported yet. Oh, the irony!");
// return;
//}
if (ActivePopulation.Species.Count == 0)
{
await this.ShowMessageAsync("Oopsie", "There are no species in this population. Please create one first.");
return;
}
CustomDialog SettingsDialog = new CustomDialog();
SettingsDialog.HorizontalContentAlignment = HorizontalAlignment.Center;
SettingsDialog.DataContext = ActivePopulation.LastRefinementOptions;
DialogRefinementSettings SettingsDialogContent = new DialogRefinementSettings();
SettingsDialogContent.DataContext = ActivePopulation.LastRefinementOptions;
SettingsDialogContent.Close += async() => await this.HideMetroDialogAsync(SettingsDialog);
SettingsDialogContent.StartRefinement += async() =>
{
await this.HideMetroDialogAsync(SettingsDialog);
ClearSpeciesDisplay(); // To avoid UI updates from refinement thread
ProcessingOptionsMPARefine Options = ActivePopulation.LastRefinementOptions;
Options.BFactorWeightingThreshold = 0.25M;
Options.DoAstigmatismDelta = Options.DoDefocus;
Options.DoAstigmatismAngle = Options.DoDefocus;
Dispatcher.Invoke(() => { SettingsDialogContent.DataContext = null; });
PerformRefinementIteration(Options);
UpdateSpeciesDisplay();
};
SettingsDialog.Content = SettingsDialogContent;
await this.ShowMetroDialogAsync(SettingsDialog);
}
static void Main(string[] args)
{
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;
//if (!Debugger.IsAttached)
// Debugger.Launch();
if (args.Length < 3)
{
return;
}
DeviceID = int.Parse(args[0]) % GPU.GetDeviceCount();
PipeName = args[1];
bool DebugMode = bool.Parse(args[2]);
GPU.SetDevice(DeviceID);
Console.WriteLine($"Running on GPU #{DeviceID} ({GPU.GetFreeMemory(DeviceID)} MB free) through {PipeName}\n");
Formatter = new BinaryFormatter();
Heartbeat = new Thread(new ThreadStart(() =>
{
if (!DebugMode)
{
while (true)
{
try
{
NamedPipeClientStream PipeHeartbeat = new NamedPipeClientStream(".", PipeName + "_heartbeat", PipeDirection.In);
PipeHeartbeat.Connect(5000);
PipeHeartbeat.Dispose();
}
catch
{
if (!Terminating)
{
Process.GetCurrentProcess().Kill();
}
}
}
}
}));
Heartbeat.Start();
while (true)
{
PipeReceive = new NamedPipeClientStream(".", PipeName + "_out", PipeDirection.In);
PipeReceive.Connect();
NamedSerializableObject Command = (NamedSerializableObject)Formatter.Deserialize(PipeReceive);
PipeReceive.Dispose();
Console.WriteLine($"Received \"{Command.Name}\", with {Command.Content.Length} arguments, for GPU #{GPU.GetDevice()}, {GPU.GetFreeMemory(DeviceID)} MB free");
try
{
Stopwatch Watch = new Stopwatch();
Watch.Start();
if (Command.Name == "Exit")
{
Movie.WriteAverageAsync?.Wait();
SendSuccessStatus(true);
Process.GetCurrentProcess().Kill();
return;
}
else if (Command.Name == "Ping")
{
Console.WriteLine("Ping!");
}
else if (Command.Name == "SetHeaderlessParams")
{
HeaderlessDims = (int2)Command.Content[0];
HeaderlessOffset = (long)Command.Content[1];
HeaderlessType = (string)Command.Content[2];
Console.WriteLine($"Set headerless parameters to {HeaderlessDims}, {HeaderlessOffset}, {HeaderlessType}");
}
else if (Command.Name == "LoadGainRef")
{
GainRef?.Dispose();
DefectMap?.Dispose();
string GainPath = (string)Command.Content[0];
bool FlipX = (bool)Command.Content[1];
bool FlipY = (bool)Command.Content[2];
bool Transpose = (bool)Command.Content[3];
string DefectsPath = (string)Command.Content[4];
if (!string.IsNullOrEmpty(GainPath))
{
GainRef = LoadAndPrepareGainReference(GainPath, FlipX, FlipY, Transpose);
}
if (!string.IsNullOrEmpty(DefectsPath))
{
DefectMap = LoadAndPrepareDefectMap(DefectsPath, FlipX, FlipY, Transpose);
}
Console.WriteLine($"Loaded gain reference and defect map: {GainRef}, {FlipX}, {FlipY}, {Transpose}, {DefectsPath}");
}
else if (Command.Name == "LoadStack")
{
OriginalStack?.Dispose();
string Path = (string)Command.Content[0];
decimal ScaleFactor = (decimal)Command.Content[1];
int EERGroupFrames = (int)Command.Content[2];
HeaderEER.GroupNFrames = EERGroupFrames;
OriginalStack = LoadAndPrepareStack(Path, ScaleFactor);
OriginalStackOwner = Helper.PathToNameWithExtension(Path);
Console.WriteLine($"Loaded stack: {OriginalStack}, {ScaleFactor}");
}
else if (Command.Name == "MovieProcessCTF")
{
string Path = (string)Command.Content[0];
ProcessingOptionsMovieCTF Options = (ProcessingOptionsMovieCTF)Command.Content[1];
if (Helper.PathToNameWithExtension(Path) != OriginalStackOwner)
{
throw new Exception("Currently loaded stack doesn't match the movie requested for processing!");
}
Movie M = new Movie(Path);
M.ProcessCTF(OriginalStack, Options);
M.SaveMeta();
Console.WriteLine($"Processed CTF for {Path}");
}
else if (Command.Name == "MovieProcessMovement")
{
string Path = (string)Command.Content[0];
ProcessingOptionsMovieMovement Options = (ProcessingOptionsMovieMovement)Command.Content[1];
if (Helper.PathToNameWithExtension(Path) != OriginalStackOwner)
{
throw new Exception("Currently loaded stack doesn't match the movie requested for processing!");
}
Movie M = new Movie(Path);
M.ProcessShift(OriginalStack, Options);
M.SaveMeta();
Console.WriteLine($"Processed movement for {Path}");
}
else if (Command.Name == "MovieExportMovie")
{
string Path = (string)Command.Content[0];
ProcessingOptionsMovieExport Options = (ProcessingOptionsMovieExport)Command.Content[1];
if (Helper.PathToNameWithExtension(Path) != OriginalStackOwner)
{
throw new Exception("Currently loaded stack doesn't match the movie requested for processing!");
}
Movie M = new Movie(Path);
M.ExportMovie(OriginalStack, Options);
M.SaveMeta();
Console.WriteLine($"Exported movie for {Path}");
}
else if (Command.Name == "MovieExportParticles")
{
string Path = (string)Command.Content[0];
ProcessingOptionsParticlesExport Options = (ProcessingOptionsParticlesExport)Command.Content[1];
float2[] Coordinates = (float2[])Command.Content[2];
if (Helper.PathToNameWithExtension(Path) != OriginalStackOwner)
{
throw new Exception("Currently loaded stack doesn't match the movie requested for processing!");
}
Movie M = new Movie(Path);
M.ExportParticles(OriginalStack, Coordinates, Options);
M.SaveMeta();
Console.WriteLine($"Exported {Coordinates.Length} particles for {Path}");
}
else if (Command.Name == "TomoProcessCTF")
{
string Path = (string)Command.Content[0];
ProcessingOptionsMovieCTF Options = (ProcessingOptionsMovieCTF)Command.Content[1];
TiltSeries T = new TiltSeries(Path);
T.ProcessCTFSimultaneous(Options);
T.SaveMeta();
Console.WriteLine($"Processed CTF for {Path}");
}
else if (Command.Name == "TomoExportParticles")
{
string Path = (string)Command.Content[0];
ProcessingOptionsTomoSubReconstruction Options = (ProcessingOptionsTomoSubReconstruction)Command.Content[1];
float3[] Coordinates = (float3[])Command.Content[2];
float3[] Angles = Command.Content[3] != null ? (float3[])Command.Content[3] : null;
TiltSeries T = new TiltSeries(Path);
T.ReconstructSubtomos(Options, Coordinates, Angles);
T.SaveMeta();
Console.WriteLine($"Exported {Coordinates.Length} particles for {Path}");
}
else if (Command.Name == "MPAPreparePopulation")
{
string Path = (string)Command.Content[0];
MPAPopulation = new Population(Path);
foreach (var species in MPAPopulation.Species)
{
Console.Write($"Preparing {species.Name} for refinement... ");
species.PrepareRefinementRequisites(true, DeviceID);
Console.WriteLine("Done.");
}
}
else if (Command.Name == "MPARefine")
{
string Path = (string)Command.Content[0];
string WorkingDirectory = (string)Command.Content[1];
string LogPath = (string)Command.Content[2];
ProcessingOptionsMPARefine Options = (ProcessingOptionsMPARefine)Command.Content[3];
DataSource Source = (DataSource)Command.Content[4];
Movie Item = null;
if (Helper.PathToExtension(Path).ToLower() == ".tomostar")
{
Item = new TiltSeries(Path);
}
else
{
Item = new Movie(Path);
}
GPU.SetDevice(DeviceID);
Item.PerformMultiParticleRefinement(WorkingDirectory, Options, MPAPopulation.Species.ToArray(), Source, GainRef, DefectMap, (message) =>
{
Console.WriteLine(message);
bool Success = false;
int Tries = 0;
while (!Success && Tries < 10)
{
try
{
using (TextWriter Writer = File.AppendText(LogPath))
Writer.WriteLine(message);
Success = true;
}
catch
{
Thread.Sleep(100);
Tries++;
}
}
});
Item.SaveMeta();
GPU.CheckGPUExceptions();
Console.WriteLine($"Finished refining {Item.Name}");
}
else if (Command.Name == "MPASaveProgress")
{
string Path = (string)Command.Content[0];
MPAPopulation.SaveRefinementProgress(Path);
}
else if (Command.Name == "TryAllocatePinnedMemory")
{
long[] ChunkSizes = (long[])Command.Content[0];
IntPtr[] Chunks = new IntPtr[ChunkSizes.Length];
for (int i = 0; i < ChunkSizes.Length; i++)
{
Chunks[i] = GPU.MallocHostPinned(ChunkSizes[i] / sizeof(float));
//Dummies.Add(Helper.ArrayOfSequence(0, (int)(ChunkSizes[i] / sizeof(float) / 2), 1));
}
GPU.CheckGPUExceptions();
//for (int i = 0; i < ChunkSizes.Length; i++)
// GPU.FreeHostPinned(Chunks[i]);
Console.WriteLine($"Successfully allocated {ChunkSizes.Sum()} bytes of pinned memory");
}
Watch.Stop();
Console.WriteLine((Watch.ElapsedMilliseconds / 1000f).ToString("F3"));
Console.WriteLine("");
SendSuccessStatus(true);
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
File.WriteAllText($"worker_{DeviceID}_crash.txt", e.ToString());
//Console.Read();
SendSuccessStatus(false);
}
}
}
private async void PerformRefinementIteration(ProcessingOptionsMPARefine options)
{
bool DoMultiProcess = true;
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount()];
string[] WorkerFolders = new string[Workers.Length];
string[] WorkerLogs = new string[Workers.Length];
foreach (var gpuID in UsedDevices)
{
WorkerFolders[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}");
Directory.CreateDirectory(WorkerFolders[gpuID]);
}
if (DoMultiProcess)
{
foreach (var gpuID in UsedDevices)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(2),
0,
"float");
WorkerLogs[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}", "run.out");
}
}
#endregion
var Progress = await this.ShowProgressAsync("Preparing for refinement – this will take a few minutes per species", "");
Progress.SetIndeterminate();
int ItemsCompleted = 0;
int ItemsToDo = ActivePopulation.Sources.Select(s => s.Files.Count).Sum();
long PinnedMemoryLimit = 1 << 30;
string[] CurrentlyRefinedItems = new string[GPU.GetDeviceCount()];
System.Timers.Timer StatusUpdater = null;
if (DoMultiProcess)
{
StatusUpdater = new System.Timers.Timer(1001);
StatusUpdater.Elapsed += (s, e) =>
{
lock (CurrentlyRefinedItems)
{
StringBuilder StatusMessage = new StringBuilder();
foreach (var gpuID in UsedDevices)
{
if (CurrentlyRefinedItems[gpuID] == null)
{
continue;
}
try
{
string ItemMessage = File.ReadLines(WorkerLogs[gpuID]).Last();
StatusMessage.Append(CurrentlyRefinedItems[gpuID] + ": " + ItemMessage + "\n");
}
catch { }
}
Dispatcher.Invoke(() => Progress.SetMessage(StatusMessage.ToString()));
}
};
}
try
{
await Task.Run(() =>
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Figuring out memory capacity..."));
WorkerWrapper[] MemoryTesters = Helper.ArrayOfFunction(i => new WorkerWrapper(0), 4);
try
{
int Tested = 0;
while (true)
{
long ChunkSize = (long)1 << 30; // 1 GB
long IncreaseBy = (long)1 << 31; // 2 GB
MemoryTesters[Tested % MemoryTesters.Length].TryAllocatePinnedMemory(Helper.ArrayOfConstant(ChunkSize, (int)(IncreaseBy / ChunkSize)));
PinnedMemoryLimit += IncreaseBy;
Tested++;
}
}
catch
{
PinnedMemoryLimit = PinnedMemoryLimit * 15 / 100; // Take 15% of that limit because Windows is weird
}
foreach (var item in MemoryTesters)
{
item.Dispose();
}
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preparing refinement requisites..."));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPAPreparePopulation(ActivePopulation.Path);
}, UsedDevices);
foreach (var species in ActivePopulation.Species)
{
species.PrepareRefinementRequisites(true, 0);
}
}
else
{
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preprocessing {species.Name}..."));
species.PrepareRefinementRequisites();
}
}
GPU.CheckGPUExceptions();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Performing refinement"));
Image.PrintObjectIDs();
if (true)
{
foreach (var source in ActivePopulation.Sources)
{
//break;
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Loading gain reference for {source.Name}..."));
Image[] GainRefs = new Image[GPU.GetDeviceCount()];
try
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
if (!string.IsNullOrEmpty(source.GainPath))
{
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose);
}
else
{
Workers[gpuID].LoadGainRef("", false, false, false);
}
}, UsedDevices);
}
else
{
Image GainRef = source.LoadAndPrepareGainReference();
if (GainRef != null)
{
GainRefs = Helper.ArrayOfFunction(i => GainRef.GetCopy(), GPU.GetDeviceCount());
}
}
}
catch
{
throw new Exception($"Could not load gain reference for {source.Name}.");
}
if (DoMultiProcess)
{
StatusUpdater.Start();
}
#region Load all items and determine pinned memory footprint for each of them
List <Movie> AllItems = source.Files.Select(pair => source.IsTiltSeries ? new TiltSeries(source.FolderPath + pair.Value) :
new Movie(source.FolderPath + pair.Value)).ToList();
Dictionary <Movie, long> ItemFootprints = new Dictionary <Movie, long>();
foreach (var item in AllItems)
{
ItemFootprints.Add(item, item.MultiParticleRefinementCalculateHostMemory(options, ActivePopulation.Species.ToArray(), source));
}
AllItems.Sort((a, b) => ItemFootprints[a].CompareTo(ItemFootprints[b]));
#endregion
long OverallFootprint = 0;
Queue <DeviceToken> Devices = new Queue <DeviceToken>();
for (int d = UsedDevices.Count - 1; d >= 0; d--)
{
Devices.Enqueue(new DeviceToken(UsedDevices[d]));
}
int NTokens = Devices.Count;
bool IsCanceled = false;
// A modified version of Helper.ForEachGPU()
int NDone = 0;
while (AllItems.Count > 0)
{
if (IsCanceled)
{
break;
}
//if (NDone++ < 200)
//{
// AllItems.RemoveAt(AllItems.Count - 1);
// continue;
//}
//if (NDone++ > 20)
// break;
while (Devices.Count <= 0)
{
Thread.Sleep(5);
}
DeviceToken CurrentDevice = null;
Movie CurrentItem = null;
while (CurrentItem == null)
{
int ItemID = AllItems.Count - 1;
lock (Devices) // Don't want OverallFootprint to change while checking
while (ItemID >= 0 && OverallFootprint + ItemFootprints[AllItems[ItemID]] > PinnedMemoryLimit)
{
ItemID--;
}
// No suitable item found and there is hope more memory will become available later
if (ItemID < 0 && OverallFootprint > 0)
{
Thread.Sleep(5);
continue;
}
// Either item can fit, or there is no hope for more memory later, so try anyway
if (ItemID < 0 && OverallFootprint == 0)
{
ItemID = AllItems.Count - 1;
}
ItemID = Math.Max(0, ItemID);
CurrentItem = AllItems[ItemID];
AllItems.Remove(CurrentItem);
lock (Devices)
{
CurrentDevice = Devices.Dequeue();
OverallFootprint += ItemFootprints[CurrentItem];
}
break;
}
Thread DeviceThread = new Thread(() =>
{
int GPUID = CurrentDevice.ID;
GPU.SetDevice(GPUID);
if (DoMultiProcess)
{
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = CurrentItem.Name;
Workers[GPUID].MPARefine(CurrentItem.Path,
WorkerFolders[GPUID],
WorkerLogs[GPUID],
options,
source);
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = null;
}
else
{
Dispatcher.InvokeAsync(() => Progress.SetTitle($"Refining {CurrentItem.Name}..."));
CurrentItem.PerformMultiParticleRefinement(WorkerFolders[GPUID], options, ActivePopulation.Species.ToArray(), source, GainRefs[GPUID], (s) =>
{
Dispatcher.InvokeAsync(() =>
{
Progress.SetMessage(s);
});
});
CurrentItem.SaveMeta();
GPU.CheckGPUExceptions();
}
Dispatcher.Invoke(() =>
{
ItemsCompleted++;
Progress.Maximum = ItemsToDo;
Progress.SetProgress(ItemsCompleted);
});
lock (Devices)
{
Devices.Enqueue(CurrentDevice);
OverallFootprint -= ItemFootprints[CurrentItem];
}
})
{
Name = $"ForEachGPU Device {CurrentDevice.ID}"
};
DeviceThread.Start();
}
while (Devices.Count != NTokens)
{
Thread.Sleep(5);
}
if (DoMultiProcess)
{
StatusUpdater.Stop();
}
source.Commit();
}
}
Image.PrintObjectIDs();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Finishing refinement"));
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage("Saving intermediate results"));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPASaveProgress(WorkerFolders[gpuID]);
Workers[gpuID].Dispose();
}, UsedDevices);
Dispatcher.InvokeAsync(() => Progress.SetMessage("Gathering intermediate results"));
ActivePopulation.GatherRefinementProgress(UsedDevices.Select(gpuID => WorkerFolders[gpuID]).ToArray());
foreach (var folder in WorkerFolders)
{
try
{
Directory.Delete(folder, true);
}
catch { }
}
}
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Reconstructing and filtering {species.Name}..."));
species.FinishRefinement();
species.Commit();
}
ActivePopulation.Save();
});
}
catch (Exception exc)
{
await Progress.CloseAsync();
await this.ShowMessageAsync("Oopsie", "Something went wrong during refinement. Sorry! Here are the details:\n\n" +
exc.ToString());
}
await Progress.CloseAsync();
}
private async void PerformRefinementIteration(ProcessingOptionsMPARefine options)
{
bool DoMultiProcess = true;
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount()];
string[] WorkerFolders = new string[Workers.Length];
string[] WorkerLogs = new string[Workers.Length];
foreach (var gpuID in UsedDevices)
{
WorkerFolders[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}");
Directory.CreateDirectory(WorkerFolders[gpuID]);
}
if (DoMultiProcess)
{
foreach (var gpuID in UsedDevices)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(2),
0,
"float");
WorkerLogs[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}", "run.out");
}
}
#endregion
var Progress = await this.ShowProgressAsync("Preparing for refinement – this will take a few minutes per species", "");
Progress.SetIndeterminate();
int ItemsCompleted = 0;
int ItemsToDo = ActivePopulation.Sources.Select(s => s.Files.Count).Sum();
string[] CurrentlyRefinedItems = new string[GPU.GetDeviceCount()];
System.Timers.Timer StatusUpdater = null;
if (DoMultiProcess)
{
StatusUpdater = new System.Timers.Timer(1001);
StatusUpdater.Elapsed += (s, e) =>
{
lock (CurrentlyRefinedItems)
{
StringBuilder StatusMessage = new StringBuilder();
foreach (var gpuID in UsedDevices)
{
if (CurrentlyRefinedItems[gpuID] == null)
{
continue;
}
try
{
string ItemMessage = File.ReadLines(WorkerLogs[gpuID]).Last();
StatusMessage.Append(CurrentlyRefinedItems[gpuID] + ": " + ItemMessage + "\n");
} catch { }
}
Dispatcher.Invoke(() => Progress.SetMessage(StatusMessage.ToString()));
}
};
}
try
{
await Task.Run(() =>
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPAPreparePopulation(ActivePopulation.Path);
}, UsedDevices);
foreach (var species in ActivePopulation.Species)
{
species.PrepareRefinementRequisites(false, 0);
}
}
else
{
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preprocessing {species.Name}..."));
species.PrepareRefinementRequisites();
}
}
GPU.CheckGPUExceptions();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Performing refinement"));
Image.PrintObjectIDs();
//if (false)
foreach (var source in ActivePopulation.Sources)
{
//break;
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Loading gain reference for {source.Name}..."));
Image[] GainRefs = new Image[GPU.GetDeviceCount()];
try
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
if (!string.IsNullOrEmpty(source.GainPath))
{
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose);
}
else
{
Workers[gpuID].LoadGainRef("", false, false, false);
}
}, UsedDevices);
}
else
{
Image GainRef = source.LoadAndPrepareGainReference();
if (GainRef != null)
{
GainRefs = Helper.ArrayOfFunction(i => GainRef.GetCopy(), GPU.GetDeviceCount());
}
}
}
catch
{
throw new Exception($"Could not load gain reference for {source.Name}.");
}
if (DoMultiProcess)
{
StatusUpdater.Start();
}
Helper.ForEachGPU(source.Files, (pair, GPUID) =>
{
if (DoMultiProcess)
{
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = pair.Value;
Workers[GPUID].MPARefine(source.FolderPath + pair.Value,
WorkerFolders[GPUID],
WorkerLogs[GPUID],
options,
source);
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = null;
}
else
{
Movie Item = null;
if (source.IsTiltSeries)
{
Item = new TiltSeries(source.FolderPath + pair.Value);
}
else
{
Item = new Movie(source.FolderPath + pair.Value);
}
Dispatcher.InvokeAsync(() => Progress.SetTitle($"Refining {Item.Name}..."));
Item.PerformMultiParticleRefinement(WorkerFolders[GPUID], options, ActivePopulation.Species.ToArray(), source, GainRefs[GPUID], (s) =>
{
Dispatcher.InvokeAsync(() =>
{
Progress.SetMessage(s);
});
});
Item.SaveMeta();
GPU.CheckGPUExceptions();
}
Dispatcher.Invoke(() =>
{
ItemsCompleted++;
Progress.Maximum = ItemsToDo;
Progress.SetProgress(ItemsCompleted);
});
return(false);
}, 1, UsedDevices);
if (DoMultiProcess)
{
StatusUpdater.Stop();
}
source.Commit();
}
Image.PrintObjectIDs();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Finishing refinement"));
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage("Saving intermediate results"));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPASaveProgress(WorkerFolders[gpuID]);
Workers[gpuID].Dispose();
}, UsedDevices);
Dispatcher.InvokeAsync(() => Progress.SetMessage("Gathering intermediate results"));
ActivePopulation.GatherRefinementProgress(UsedDevices.Select(gpuID => WorkerFolders[gpuID]).ToArray());
foreach (var folder in WorkerFolders)
{
try
{
Directory.Delete(folder, true);
}
catch { }
}
}
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Reconstructing and filtering {species.Name}..."));
species.FinishRefinement();
species.Commit();
}
});
}
catch (Exception exc)
{
await Progress.CloseAsync();
await this.ShowMessageAsync("Oopsie", "Something went wrong during refinement. Sorry! Here are the details:\n\n" +
exc.ToString());
}
await Progress.CloseAsync();
}
private async void ButtonPopulationRefine_Click(object sender, RoutedEventArgs e)
{
if (ActivePopulation == null)
{
return;
}
//if (ActivePopulation.Species.Count > 1)
//{
// await this.ShowMessageAsync("Oopsie", "Multiple species aren't supported yet. Oh, the irony!");
// return;
//}
if (ActivePopulation.Species.Count == 0)
{
await this.ShowMessageAsync("Oopsie", "There are no species in this population. Please create one first.");
return;
}
CustomDialog SettingsDialog = new CustomDialog();
SettingsDialog.HorizontalContentAlignment = HorizontalAlignment.Center;
DialogRefinementSettings SettingsDialogContent = new DialogRefinementSettings();
SettingsDialogContent.Close += async() => await this.HideMetroDialogAsync(SettingsDialog);
SettingsDialogContent.StartRefinement += async() =>
{
await this.HideMetroDialogAsync(SettingsDialog);
ClearSpeciesDisplay(); // To avoid UI updates from refinement thread
ProcessingOptionsMPARefine Options = new ProcessingOptionsMPARefine();
Options.NIterations = (int)SettingsDialogContent.SliderNIterations.Value;
Options.MinimumCTFRefinementResolution = (float)SettingsDialogContent.SliderCTFResolution.Value;
Options.ImageWarpResolution = new int2((int)SettingsDialogContent.SliderImageWarpWidth.Value,
(int)SettingsDialogContent.SliderImageWarpHeight.Value);
Options.VolumeWarpResolution = new int4((int)SettingsDialogContent.SliderVolumeWarpWidth.Value,
(int)SettingsDialogContent.SliderVolumeWarpHeight.Value,
(int)SettingsDialogContent.SliderVolumeWarpDepth.Value,
(int)SettingsDialogContent.SliderVolumeWarpDuration.Value);
Options.RefineMovies = (bool)SettingsDialogContent.CheckRefineTiltMovies.IsChecked;
Options.RefinedComponentsWarp = 0;
if ((bool)SettingsDialogContent.CheckImageWarp.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.ImageWarp;
}
if ((bool)SettingsDialogContent.CheckVolumeWarp.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.VolumeWarp;
}
if ((bool)SettingsDialogContent.CheckStageAngles.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.AxisAngle;
}
if ((bool)SettingsDialogContent.CheckParticlePositions.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.ParticlePosition;
}
if ((bool)SettingsDialogContent.CheckParticleAngles.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.ParticleAngle;
}
if ((bool)SettingsDialogContent.CheckBeamTilt.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.BeamTilt;
}
if ((bool)SettingsDialogContent.CheckMagnification.IsChecked)
{
Options.RefinedComponentsWarp |= WarpOptimizationTypes.Magnification;
}
Options.RefinedComponentsCTF = 0;
if ((bool)SettingsDialogContent.CheckDefocus.IsChecked)
{
Options.RefinedComponentsCTF |= CTFOptimizationTypes.Defocus;
Options.RefinedComponentsCTF |= CTFOptimizationTypes.AstigmatismDelta;
Options.RefinedComponentsCTF |= CTFOptimizationTypes.AstigmatismAngle;
}
if ((bool)SettingsDialogContent.CheckPhaseShift.IsChecked)
{
Options.RefinedComponentsCTF |= CTFOptimizationTypes.PhaseShift;
}
if ((bool)SettingsDialogContent.CheckBeamTilt.IsChecked)
{
Options.RefinedComponentsCTF |= CTFOptimizationTypes.BeamTilt;
}
if ((bool)SettingsDialogContent.CheckMagnification.IsChecked)
{
Options.RefinedComponentsCTF |= CTFOptimizationTypes.PixelSize;
}
PerformRefinementIteration(Options);
UpdateSpeciesDisplay();
};
SettingsDialog.Content = SettingsDialogContent;
await this.ShowMetroDialogAsync(SettingsDialog);
}
public void Load(string path)
{
DoAutosave = false;
Path = path;
using (Stream SettingsStream = File.OpenRead(path))
{
XPathDocument Doc = new XPathDocument(SettingsStream);
XPathNavigator Reader = Doc.CreateNavigator();
Reader.MoveToRoot();
Reader.MoveToChild("Population", "");
ReadFromXML(Reader);
List <Species> AllSpecies = new List <Species>();
foreach (XPathNavigator nav in Reader.Select("Species/Species"))
{
Guid SpeciesGUID = Guid.Parse(nav.GetAttribute("GUID", ""));
string SpeciesPath = nav.GetAttribute("Path", "");
Species LoadedSpecies = Sociology.Species.FromFile(SpeciesPath);
if (LoadedSpecies.GUID != SpeciesGUID)
{
throw new Exception("Stored GUID does not match that of the species.");
}
AllSpecies.Add(LoadedSpecies);
}
foreach (var species in AllSpecies)
{
species.ResolveChildren(AllSpecies);
}
Species.Clear();
foreach (var toplevel in AllSpecies.Where(s => s.Parent == null))
{
Species.Add(toplevel);
}
foreach (XPathNavigator nav in Reader.Select("Sources/Source"))
{
string Path = nav.GetAttribute("Path", "");
Guid SourceGUID = Guid.Parse(nav.GetAttribute("GUID", ""));
DataSource LoadedSource = DataSource.FromFile(Path);
if (SourceGUID != LoadedSource.GUID)
{
throw new Exception("Stored GUID does not match that of the data source.");
}
Sources.Add(LoadedSource);
}
XPathNavigator NavLastRefinementOptions = Reader.SelectSingleNode("//LastRefinementOptions");
if (NavLastRefinementOptions != null)
{
ProcessingOptionsMPARefine Temp = new ProcessingOptionsMPARefine();
Temp.ReadFromXML(NavLastRefinementOptions);
LastRefinementOptions = Temp;
}
}
DoAutosave = true;
}
