private void StopClient()
{
try
{
_zeroConfig?.Stop();
_zeroConfig = null;
NowPlayingUpdater.Stop();
StatusUpdater.Stop();
_mpMessageHandler.UnsubscribeFromMessages();
}
catch (Exception ex)
{
ServiceRegistration.Get <ILogger>().Error("WifiRemote: Failed to stop server", ex);
}
}
private async void PerformRefinementIteration(ProcessingOptionsMPARefine options)
{
bool DoMultiProcess = true;
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount()];
string[] WorkerFolders = new string[Workers.Length];
string[] WorkerLogs = new string[Workers.Length];
foreach (var gpuID in UsedDevices)
{
WorkerFolders[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}");
Directory.CreateDirectory(WorkerFolders[gpuID]);
}
if (DoMultiProcess)
{
foreach (var gpuID in UsedDevices)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(2),
0,
"float");
WorkerLogs[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}", "run.out");
}
}
#endregion
var Progress = await this.ShowProgressAsync("Preparing for refinement – this will take a few minutes per species", "");
Progress.SetIndeterminate();
int ItemsCompleted = 0;
int ItemsToDo = ActivePopulation.Sources.Select(s => s.Files.Count).Sum();
long PinnedMemoryLimit = 1 << 30;
string[] CurrentlyRefinedItems = new string[GPU.GetDeviceCount()];
System.Timers.Timer StatusUpdater = null;
if (DoMultiProcess)
{
StatusUpdater = new System.Timers.Timer(1001);
StatusUpdater.Elapsed += (s, e) =>
{
lock (CurrentlyRefinedItems)
{
StringBuilder StatusMessage = new StringBuilder();
foreach (var gpuID in UsedDevices)
{
if (CurrentlyRefinedItems[gpuID] == null)
{
continue;
}
try
{
string ItemMessage = File.ReadLines(WorkerLogs[gpuID]).Last();
StatusMessage.Append(CurrentlyRefinedItems[gpuID] + ": " + ItemMessage + "\n");
}
catch { }
}
Dispatcher.Invoke(() => Progress.SetMessage(StatusMessage.ToString()));
}
};
}
try
{
await Task.Run(() =>
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Figuring out memory capacity..."));
WorkerWrapper[] MemoryTesters = Helper.ArrayOfFunction(i => new WorkerWrapper(0), 4);
try
{
int Tested = 0;
while (true)
{
long ChunkSize = (long)1 << 30; // 1 GB
long IncreaseBy = (long)1 << 31; // 2 GB
MemoryTesters[Tested % MemoryTesters.Length].TryAllocatePinnedMemory(Helper.ArrayOfConstant(ChunkSize, (int)(IncreaseBy / ChunkSize)));
PinnedMemoryLimit += IncreaseBy;
Tested++;
}
}
catch
{
PinnedMemoryLimit = PinnedMemoryLimit * 15 / 100; // Take 15% of that limit because Windows is weird
}
foreach (var item in MemoryTesters)
{
item.Dispose();
}
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preparing refinement requisites..."));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPAPreparePopulation(ActivePopulation.Path);
}, UsedDevices);
foreach (var species in ActivePopulation.Species)
{
species.PrepareRefinementRequisites(true, 0);
}
}
else
{
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preprocessing {species.Name}..."));
species.PrepareRefinementRequisites();
}
}
GPU.CheckGPUExceptions();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Performing refinement"));
Image.PrintObjectIDs();
if (true)
{
foreach (var source in ActivePopulation.Sources)
{
//break;
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Loading gain reference for {source.Name}..."));
Image[] GainRefs = new Image[GPU.GetDeviceCount()];
try
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
if (!string.IsNullOrEmpty(source.GainPath))
{
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose);
}
else
{
Workers[gpuID].LoadGainRef("", false, false, false);
}
}, UsedDevices);
}
else
{
Image GainRef = source.LoadAndPrepareGainReference();
if (GainRef != null)
{
GainRefs = Helper.ArrayOfFunction(i => GainRef.GetCopy(), GPU.GetDeviceCount());
}
}
}
catch
{
throw new Exception($"Could not load gain reference for {source.Name}.");
}
if (DoMultiProcess)
{
StatusUpdater.Start();
}
#region Load all items and determine pinned memory footprint for each of them
List <Movie> AllItems = source.Files.Select(pair => source.IsTiltSeries ? new TiltSeries(source.FolderPath + pair.Value) :
new Movie(source.FolderPath + pair.Value)).ToList();
Dictionary <Movie, long> ItemFootprints = new Dictionary <Movie, long>();
foreach (var item in AllItems)
{
ItemFootprints.Add(item, item.MultiParticleRefinementCalculateHostMemory(options, ActivePopulation.Species.ToArray(), source));
}
AllItems.Sort((a, b) => ItemFootprints[a].CompareTo(ItemFootprints[b]));
#endregion
long OverallFootprint = 0;
Queue <DeviceToken> Devices = new Queue <DeviceToken>();
for (int d = UsedDevices.Count - 1; d >= 0; d--)
{
Devices.Enqueue(new DeviceToken(UsedDevices[d]));
}
int NTokens = Devices.Count;
bool IsCanceled = false;
// A modified version of Helper.ForEachGPU()
int NDone = 0;
while (AllItems.Count > 0)
{
if (IsCanceled)
{
break;
}
//if (NDone++ < 200)
//{
// AllItems.RemoveAt(AllItems.Count - 1);
// continue;
//}
//if (NDone++ > 20)
// break;
while (Devices.Count <= 0)
{
Thread.Sleep(5);
}
DeviceToken CurrentDevice = null;
Movie CurrentItem = null;
while (CurrentItem == null)
{
int ItemID = AllItems.Count - 1;
lock (Devices) // Don't want OverallFootprint to change while checking
while (ItemID >= 0 && OverallFootprint + ItemFootprints[AllItems[ItemID]] > PinnedMemoryLimit)
{
ItemID--;
}
// No suitable item found and there is hope more memory will become available later
if (ItemID < 0 && OverallFootprint > 0)
{
Thread.Sleep(5);
continue;
}
// Either item can fit, or there is no hope for more memory later, so try anyway
if (ItemID < 0 && OverallFootprint == 0)
{
ItemID = AllItems.Count - 1;
}
ItemID = Math.Max(0, ItemID);
CurrentItem = AllItems[ItemID];
AllItems.Remove(CurrentItem);
lock (Devices)
{
CurrentDevice = Devices.Dequeue();
OverallFootprint += ItemFootprints[CurrentItem];
}
break;
}
Thread DeviceThread = new Thread(() =>
{
int GPUID = CurrentDevice.ID;
GPU.SetDevice(GPUID);
if (DoMultiProcess)
{
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = CurrentItem.Name;
Workers[GPUID].MPARefine(CurrentItem.Path,
WorkerFolders[GPUID],
WorkerLogs[GPUID],
options,
source);
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = null;
}
else
{
Dispatcher.InvokeAsync(() => Progress.SetTitle($"Refining {CurrentItem.Name}..."));
CurrentItem.PerformMultiParticleRefinement(WorkerFolders[GPUID], options, ActivePopulation.Species.ToArray(), source, GainRefs[GPUID], (s) =>
{
Dispatcher.InvokeAsync(() =>
{
Progress.SetMessage(s);
});
});
CurrentItem.SaveMeta();
GPU.CheckGPUExceptions();
}
Dispatcher.Invoke(() =>
{
ItemsCompleted++;
Progress.Maximum = ItemsToDo;
Progress.SetProgress(ItemsCompleted);
});
lock (Devices)
{
Devices.Enqueue(CurrentDevice);
OverallFootprint -= ItemFootprints[CurrentItem];
}
})
{
Name = $"ForEachGPU Device {CurrentDevice.ID}"
};
DeviceThread.Start();
}
while (Devices.Count != NTokens)
{
Thread.Sleep(5);
}
if (DoMultiProcess)
{
StatusUpdater.Stop();
}
source.Commit();
}
}
Image.PrintObjectIDs();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Finishing refinement"));
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage("Saving intermediate results"));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPASaveProgress(WorkerFolders[gpuID]);
Workers[gpuID].Dispose();
}, UsedDevices);
Dispatcher.InvokeAsync(() => Progress.SetMessage("Gathering intermediate results"));
ActivePopulation.GatherRefinementProgress(UsedDevices.Select(gpuID => WorkerFolders[gpuID]).ToArray());
foreach (var folder in WorkerFolders)
{
try
{
Directory.Delete(folder, true);
}
catch { }
}
}
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Reconstructing and filtering {species.Name}..."));
species.FinishRefinement();
species.Commit();
}
ActivePopulation.Save();
});
}
catch (Exception exc)
{
await Progress.CloseAsync();
await this.ShowMessageAsync("Oopsie", "Something went wrong during refinement. Sorry! Here are the details:\n\n" +
exc.ToString());
}
await Progress.CloseAsync();
}
private async void PerformRefinementIteration(ProcessingOptionsMPARefine options)
{
bool DoMultiProcess = true;
#region Create worker processes
int NDevices = GPU.GetDeviceCount();
List <int> UsedDevices = Options.MainWindow.GetDeviceList();
WorkerWrapper[] Workers = new WorkerWrapper[GPU.GetDeviceCount()];
string[] WorkerFolders = new string[Workers.Length];
string[] WorkerLogs = new string[Workers.Length];
foreach (var gpuID in UsedDevices)
{
WorkerFolders[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}");
Directory.CreateDirectory(WorkerFolders[gpuID]);
}
if (DoMultiProcess)
{
foreach (var gpuID in UsedDevices)
{
Workers[gpuID] = new WorkerWrapper(gpuID);
Workers[gpuID].SetHeaderlessParams(new int2(2),
0,
"float");
WorkerLogs[gpuID] = System.IO.Path.Combine(ActivePopulation.FolderPath, "refinement_temp", $"worker{gpuID}", "run.out");
}
}
#endregion
var Progress = await this.ShowProgressAsync("Preparing for refinement – this will take a few minutes per species", "");
Progress.SetIndeterminate();
int ItemsCompleted = 0;
int ItemsToDo = ActivePopulation.Sources.Select(s => s.Files.Count).Sum();
string[] CurrentlyRefinedItems = new string[GPU.GetDeviceCount()];
System.Timers.Timer StatusUpdater = null;
if (DoMultiProcess)
{
StatusUpdater = new System.Timers.Timer(1001);
StatusUpdater.Elapsed += (s, e) =>
{
lock (CurrentlyRefinedItems)
{
StringBuilder StatusMessage = new StringBuilder();
foreach (var gpuID in UsedDevices)
{
if (CurrentlyRefinedItems[gpuID] == null)
{
continue;
}
try
{
string ItemMessage = File.ReadLines(WorkerLogs[gpuID]).Last();
StatusMessage.Append(CurrentlyRefinedItems[gpuID] + ": " + ItemMessage + "\n");
} catch { }
}
Dispatcher.Invoke(() => Progress.SetMessage(StatusMessage.ToString()));
}
};
}
try
{
await Task.Run(() =>
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPAPreparePopulation(ActivePopulation.Path);
}, UsedDevices);
foreach (var species in ActivePopulation.Species)
{
species.PrepareRefinementRequisites(false, 0);
}
}
else
{
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Preprocessing {species.Name}..."));
species.PrepareRefinementRequisites();
}
}
GPU.CheckGPUExceptions();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Performing refinement"));
Image.PrintObjectIDs();
//if (false)
foreach (var source in ActivePopulation.Sources)
{
//break;
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Loading gain reference for {source.Name}..."));
Image[] GainRefs = new Image[GPU.GetDeviceCount()];
try
{
if (DoMultiProcess)
{
Helper.ForEachGPUOnce(gpuID =>
{
if (!string.IsNullOrEmpty(source.GainPath))
{
Workers[gpuID].LoadGainRef(source.GainPath,
source.GainFlipX,
source.GainFlipY,
source.GainTranspose);
}
else
{
Workers[gpuID].LoadGainRef("", false, false, false);
}
}, UsedDevices);
}
else
{
Image GainRef = source.LoadAndPrepareGainReference();
if (GainRef != null)
{
GainRefs = Helper.ArrayOfFunction(i => GainRef.GetCopy(), GPU.GetDeviceCount());
}
}
}
catch
{
throw new Exception($"Could not load gain reference for {source.Name}.");
}
if (DoMultiProcess)
{
StatusUpdater.Start();
}
Helper.ForEachGPU(source.Files, (pair, GPUID) =>
{
if (DoMultiProcess)
{
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = pair.Value;
Workers[GPUID].MPARefine(source.FolderPath + pair.Value,
WorkerFolders[GPUID],
WorkerLogs[GPUID],
options,
source);
lock (CurrentlyRefinedItems)
CurrentlyRefinedItems[GPUID] = null;
}
else
{
Movie Item = null;
if (source.IsTiltSeries)
{
Item = new TiltSeries(source.FolderPath + pair.Value);
}
else
{
Item = new Movie(source.FolderPath + pair.Value);
}
Dispatcher.InvokeAsync(() => Progress.SetTitle($"Refining {Item.Name}..."));
Item.PerformMultiParticleRefinement(WorkerFolders[GPUID], options, ActivePopulation.Species.ToArray(), source, GainRefs[GPUID], (s) =>
{
Dispatcher.InvokeAsync(() =>
{
Progress.SetMessage(s);
});
});
Item.SaveMeta();
GPU.CheckGPUExceptions();
}
Dispatcher.Invoke(() =>
{
ItemsCompleted++;
Progress.Maximum = ItemsToDo;
Progress.SetProgress(ItemsCompleted);
});
return(false);
}, 1, UsedDevices);
if (DoMultiProcess)
{
StatusUpdater.Stop();
}
source.Commit();
}
Image.PrintObjectIDs();
Dispatcher.InvokeAsync(() => Progress.SetTitle("Finishing refinement"));
if (DoMultiProcess)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage("Saving intermediate results"));
Helper.ForEachGPUOnce(gpuID =>
{
Workers[gpuID].MPASaveProgress(WorkerFolders[gpuID]);
Workers[gpuID].Dispose();
}, UsedDevices);
Dispatcher.InvokeAsync(() => Progress.SetMessage("Gathering intermediate results"));
ActivePopulation.GatherRefinementProgress(UsedDevices.Select(gpuID => WorkerFolders[gpuID]).ToArray());
foreach (var folder in WorkerFolders)
{
try
{
Directory.Delete(folder, true);
}
catch { }
}
}
foreach (var species in ActivePopulation.Species)
{
Dispatcher.InvokeAsync(() => Progress.SetMessage($"Reconstructing and filtering {species.Name}..."));
species.FinishRefinement();
species.Commit();
}
});
}
catch (Exception exc)
{
await Progress.CloseAsync();
await this.ShowMessageAsync("Oopsie", "Something went wrong during refinement. Sorry! Here are the details:\n\n" +
exc.ToString());
}
await Progress.CloseAsync();
}