private IDictionary<int, string> m_usernames; // Always update this when updating m_trainingData. Is null when rec sources are finalized.
#endregion Fields
#region Constructors
/// <summary>
///
/// </summary>
/// <param name="trainingDataLoaderFactory">Must be thread-safe.</param>
public RecServiceState(IMalTrainingDataLoaderFactory trainingDataLoaderFactory)
{
m_trainingDataLoaderFactory = trainingDataLoaderFactory;
JsonRecSourceTypes = GetJsonRecSourceTypes();
using (IMalTrainingDataLoader trainingDataLoader = trainingDataLoaderFactory.GetTrainingDataLoader())
{
m_trainingData = LoadTrainingDataOnInit(trainingDataLoader);
m_usernames = GetUsernamesFromTrainingData(m_trainingData);
m_animes = m_trainingData.Animes;
m_prereqs = LoadPrereqsOnInit(trainingDataLoader);
}
m_trainingDataLock = new ReaderWriterLockSlim();
m_recSourcesLock = new ReaderWriterLockSlim();
}
// Loads training data and prerequisites from the database in parallel and does not return until they are loaded.
private static (MalTrainingData trainingData, IDictionary <int, IList <int> > prereqs) LoadInitialData(IMalTrainingDataLoaderFactory trainingDataLoaderFactory, CancellationToken serviceStopToken)
{
using (IMalTrainingDataLoader initialTrainingDataLoader = trainingDataLoaderFactory.GetTrainingDataLoader())
using (CancellationTokenSource trainingDataOtherFaultOrCancellation = new CancellationTokenSource())
using (CancellationTokenSource trainingDataCancel = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, trainingDataOtherFaultOrCancellation.Token))
using (CancellationTokenSource prereqsOtherFaultOrCancellation = new CancellationTokenSource())
using (CancellationTokenSource prereqsCancel = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, prereqsOtherFaultOrCancellation.Token))
{
CancellableAsyncFunc <MalTrainingData> trainingDataAsyncFunc = new CancellableAsyncFunc <MalTrainingData>(
() => LoadTrainingDataOnInitAsync(initialTrainingDataLoader, trainingDataCancel.Token), trainingDataOtherFaultOrCancellation);
CancellableTask <MalTrainingData> trainingDataTask = trainingDataAsyncFunc.StartTaskEnsureExceptionsWrapped();
CancellableAsyncFunc <IDictionary <int, IList <int> > > prereqsAsyncFunc = new CancellableAsyncFunc <IDictionary <int, IList <int> > >(
() => LoadPrereqsOnInit(initialTrainingDataLoader, prereqsCancel.Token), prereqsOtherFaultOrCancellation);
CancellableTask <IDictionary <int, IList <int> > > prereqsTask = prereqsAsyncFunc.StartTaskEnsureExceptionsWrapped();
AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(trainingDataTask, prereqsTask).ConfigureAwait(false).GetAwaiter().GetResult();
return(trainingDataTask.Task.Result, prereqsTask.Task.Result);
}
}
private async Task ReloadTrainingDataLowMemoryAsync(bool finalize, CancellationToken cancellationToken)
{
using (var trainingDataWriteLock = await m_trainingDataLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false))
using (var recSourcesWriteLock = await m_recSourcesLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false))
{
Logging.Log.Info("Reloading training data and prerequisites and retraining rec sources. Rec sources will not be available until retraining all rec sources is complete.");
Stopwatch totalTimer = Stopwatch.StartNew();
m_recSources.Clear();
m_trainingData = null;
m_usernames = null;
m_animes = null;
m_prereqs = null;
m_finalized = false;
GC.Collect();
Logging.Log.Info("Rec sources cleared.");
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
// Load new training data
// If this throws an error, m_trainingData is left null. Methods that use m_trainingData should check it for null.
using (IMalTrainingDataLoader malTrainingDataLoader = m_trainingDataLoaderFactory.GetTrainingDataLoader())
using (CancellationTokenSource faultCanceler = new CancellationTokenSource())
using (CancellationTokenSource faultOrUserCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, faultCanceler.Token))
{
Stopwatch trainingDataTimer = Stopwatch.StartNew();
CancellableTask <MalTrainingData> trainingDataTask = new CancellableTask <MalTrainingData>(
malTrainingDataLoader.LoadMalTrainingDataAsync(faultOrUserCancel.Token), faultCanceler);
Task trainingDataTimerTask = trainingDataTask.Task.ContinueWith(task =>
{
trainingDataTimer.Stop();
Logging.Log.InfoFormat("Training data loaded. {0} users, {1} animes, {2} entries. Took {3}.",
task.Result.Users.Count, task.Result.Animes.Count,
task.Result.Users.Keys.Sum(userId => task.Result.Users[userId].Entries.Count),
trainingDataTimer.Elapsed);
},
cancellationToken, TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnCanceled
| TaskContinuationOptions.NotOnFaulted, TaskScheduler.Current);
Stopwatch prereqsTimer = Stopwatch.StartNew();
CancellableTask <IDictionary <int, IList <int> > > prereqsTask = new CancellableTask <IDictionary <int, IList <int> > >(
malTrainingDataLoader.LoadPrerequisitesAsync(faultOrUserCancel.Token), faultCanceler);
Task prereqsTimerTask = prereqsTask.Task.ContinueWith(task =>
{
prereqsTimer.Stop();
int numPrereqs = task.Result.Values.Sum(prereqList => prereqList.Count);
Logging.Log.InfoFormat("Prerequisites loaded. {0} prerequisites for {1} animes. Took {2}.",
numPrereqs, task.Result.Count, prereqsTimer.Elapsed);
},
cancellationToken, TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnCanceled
| TaskContinuationOptions.NotOnFaulted, TaskScheduler.Current);
await AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(trainingDataTask, prereqsTask).ConfigureAwait(false);
m_trainingData = trainingDataTask.Task.Result;
m_usernames = GetUsernamesFromTrainingData(m_trainingData);
m_animes = m_trainingData.Animes;
m_prereqs = prereqsTask.Task.Result;
await trainingDataTimerTask.ConfigureAwait(false);
await prereqsTimerTask.ConfigureAwait(false);
}
GC.Collect();
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
// Then retrain all loaded rec sources.
if (m_recSourceFactories.Count == 0)
{
Logging.Log.Info("No rec sources to retrain.");
}
else
{
Logging.Log.Info("Retraining rec sources.");
object recSourcesLockAndMemFence = new object();
List <Task> recSourceTrainTasksList = new List <Task>();
// ToList() so we can unload a rec source as we iterate if it errors while training.
foreach (string recSourceNameLoopVar in m_recSourceFactories.Keys.ToList())
{
string recSourceName = recSourceNameLoopVar; // avoid capturing the loop var
ITrainableJsonRecSource recSource = m_recSourceFactories[recSourceName]();
Task recSourceTrainTask = Task.Run(() =>
{
Logging.Log.InfoFormat("Retraining rec source {0} ({1}).", recSourceName, recSource);
Stopwatch trainTimer = Stopwatch.StartNew();
try
{
recSource.Train(m_trainingData, m_usernames, cancellationToken);
trainTimer.Stop();
Logging.Log.InfoFormat("Trained rec source {0} ({1}). Took {2}.", recSourceName, recSource, trainTimer.Elapsed);
lock (recSourcesLockAndMemFence)
{
m_recSources[recSourceName] = recSource;
}
}
catch (OperationCanceledException)
{
Logging.Log.InfoFormat("Canceled while retraining rec source {0} ({1}). Unloading it.", recSourceName, recSource);
lock (recSourcesLockAndMemFence)
{
m_recSourceFactories.Remove(recSourceName);
}
throw;
}
catch (Exception ex)
{
Logging.Log.ErrorFormat("Error retraining rec source {0} ({1}): {2} Unloading it.",
ex, recSourceName, recSource, ex.Message);
lock (recSourcesLockAndMemFence)
{
m_recSourceFactories.Remove(recSourceName);
}
}
}, cancellationToken);
recSourceTrainTasksList.Add(recSourceTrainTask);
}
// Wait for all to complete or cancellation. There should not be any exceptions other than OperationCanceledException.
await Task.WhenAll(recSourceTrainTasksList).ConfigureAwait(false);
lock (recSourcesLockAndMemFence)
{
; // just for the fence
}
}
if (finalize)
{
m_trainingData = null;
m_usernames = null;
m_finalized = true;
Logging.Log.Info("Finalized rec sources.");
}
totalTimer.Stop();
Logging.Log.InfoFormat("All rec sources retrained with the latest data. Total time: {0}", totalTimer.Elapsed);
}
GC.Collect();
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
}