private IDictionary<int, string> m_usernames; // Always update this when updating m_trainingData. Is null when rec sources are finalized. #endregion Fields #region Constructors /// <summary> /// /// </summary> /// <param name="trainingDataLoaderFactory">Must be thread-safe.</param> public RecServiceState(IMalTrainingDataLoaderFactory trainingDataLoaderFactory) { m_trainingDataLoaderFactory = trainingDataLoaderFactory; JsonRecSourceTypes = GetJsonRecSourceTypes(); using (IMalTrainingDataLoader trainingDataLoader = trainingDataLoaderFactory.GetTrainingDataLoader()) { m_trainingData = LoadTrainingDataOnInit(trainingDataLoader); m_usernames = GetUsernamesFromTrainingData(m_trainingData); m_animes = m_trainingData.Animes; m_prereqs = LoadPrereqsOnInit(trainingDataLoader); } m_trainingDataLock = new ReaderWriterLockSlim(); m_recSourcesLock = new ReaderWriterLockSlim(); }
// Loads training data and prerequisites from the database in parallel and does not return until they are loaded. private static (MalTrainingData trainingData, IDictionary <int, IList <int> > prereqs) LoadInitialData(IMalTrainingDataLoaderFactory trainingDataLoaderFactory, CancellationToken serviceStopToken) { using (IMalTrainingDataLoader initialTrainingDataLoader = trainingDataLoaderFactory.GetTrainingDataLoader()) using (CancellationTokenSource trainingDataOtherFaultOrCancellation = new CancellationTokenSource()) using (CancellationTokenSource trainingDataCancel = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, trainingDataOtherFaultOrCancellation.Token)) using (CancellationTokenSource prereqsOtherFaultOrCancellation = new CancellationTokenSource()) using (CancellationTokenSource prereqsCancel = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, prereqsOtherFaultOrCancellation.Token)) { CancellableAsyncFunc <MalTrainingData> trainingDataAsyncFunc = new CancellableAsyncFunc <MalTrainingData>( () => LoadTrainingDataOnInitAsync(initialTrainingDataLoader, trainingDataCancel.Token), trainingDataOtherFaultOrCancellation); CancellableTask <MalTrainingData> trainingDataTask = trainingDataAsyncFunc.StartTaskEnsureExceptionsWrapped(); CancellableAsyncFunc <IDictionary <int, IList <int> > > prereqsAsyncFunc = new CancellableAsyncFunc <IDictionary <int, IList <int> > >( () => LoadPrereqsOnInit(initialTrainingDataLoader, prereqsCancel.Token), prereqsOtherFaultOrCancellation); CancellableTask <IDictionary <int, IList <int> > > prereqsTask = prereqsAsyncFunc.StartTaskEnsureExceptionsWrapped(); AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(trainingDataTask, prereqsTask).ConfigureAwait(false).GetAwaiter().GetResult(); return(trainingDataTask.Task.Result, prereqsTask.Task.Result); } }
private async Task ReloadTrainingDataLowMemoryAsync(bool finalize, CancellationToken cancellationToken) { using (var trainingDataWriteLock = await m_trainingDataLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false)) using (var recSourcesWriteLock = await m_recSourcesLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false)) { Logging.Log.Info("Reloading training data and prerequisites and retraining rec sources. Rec sources will not be available until retraining all rec sources is complete."); Stopwatch totalTimer = Stopwatch.StartNew(); m_recSources.Clear(); m_trainingData = null; m_usernames = null; m_animes = null; m_prereqs = null; m_finalized = false; GC.Collect(); Logging.Log.Info("Rec sources cleared."); Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false)); // Load new training data // If this throws an error, m_trainingData is left null. Methods that use m_trainingData should check it for null. using (IMalTrainingDataLoader malTrainingDataLoader = m_trainingDataLoaderFactory.GetTrainingDataLoader()) using (CancellationTokenSource faultCanceler = new CancellationTokenSource()) using (CancellationTokenSource faultOrUserCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, faultCanceler.Token)) { Stopwatch trainingDataTimer = Stopwatch.StartNew(); CancellableTask <MalTrainingData> trainingDataTask = new CancellableTask <MalTrainingData>( malTrainingDataLoader.LoadMalTrainingDataAsync(faultOrUserCancel.Token), faultCanceler); Task trainingDataTimerTask = trainingDataTask.Task.ContinueWith(task => { trainingDataTimer.Stop(); Logging.Log.InfoFormat("Training data loaded. {0} users, {1} animes, {2} entries. Took {3}.", task.Result.Users.Count, task.Result.Animes.Count, task.Result.Users.Keys.Sum(userId => task.Result.Users[userId].Entries.Count), trainingDataTimer.Elapsed); }, cancellationToken, TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnCanceled | TaskContinuationOptions.NotOnFaulted, TaskScheduler.Current); Stopwatch prereqsTimer = Stopwatch.StartNew(); CancellableTask <IDictionary <int, IList <int> > > prereqsTask = new CancellableTask <IDictionary <int, IList <int> > >( malTrainingDataLoader.LoadPrerequisitesAsync(faultOrUserCancel.Token), faultCanceler); Task prereqsTimerTask = prereqsTask.Task.ContinueWith(task => { prereqsTimer.Stop(); int numPrereqs = task.Result.Values.Sum(prereqList => prereqList.Count); Logging.Log.InfoFormat("Prerequisites loaded. {0} prerequisites for {1} animes. Took {2}.", numPrereqs, task.Result.Count, prereqsTimer.Elapsed); }, cancellationToken, TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnCanceled | TaskContinuationOptions.NotOnFaulted, TaskScheduler.Current); await AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(trainingDataTask, prereqsTask).ConfigureAwait(false); m_trainingData = trainingDataTask.Task.Result; m_usernames = GetUsernamesFromTrainingData(m_trainingData); m_animes = m_trainingData.Animes; m_prereqs = prereqsTask.Task.Result; await trainingDataTimerTask.ConfigureAwait(false); await prereqsTimerTask.ConfigureAwait(false); } GC.Collect(); Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false)); // Then retrain all loaded rec sources. if (m_recSourceFactories.Count == 0) { Logging.Log.Info("No rec sources to retrain."); } else { Logging.Log.Info("Retraining rec sources."); object recSourcesLockAndMemFence = new object(); List <Task> recSourceTrainTasksList = new List <Task>(); // ToList() so we can unload a rec source as we iterate if it errors while training. foreach (string recSourceNameLoopVar in m_recSourceFactories.Keys.ToList()) { string recSourceName = recSourceNameLoopVar; // avoid capturing the loop var ITrainableJsonRecSource recSource = m_recSourceFactories[recSourceName](); Task recSourceTrainTask = Task.Run(() => { Logging.Log.InfoFormat("Retraining rec source {0} ({1}).", recSourceName, recSource); Stopwatch trainTimer = Stopwatch.StartNew(); try { recSource.Train(m_trainingData, m_usernames, cancellationToken); trainTimer.Stop(); Logging.Log.InfoFormat("Trained rec source {0} ({1}). Took {2}.", recSourceName, recSource, trainTimer.Elapsed); lock (recSourcesLockAndMemFence) { m_recSources[recSourceName] = recSource; } } catch (OperationCanceledException) { Logging.Log.InfoFormat("Canceled while retraining rec source {0} ({1}). Unloading it.", recSourceName, recSource); lock (recSourcesLockAndMemFence) { m_recSourceFactories.Remove(recSourceName); } throw; } catch (Exception ex) { Logging.Log.ErrorFormat("Error retraining rec source {0} ({1}): {2} Unloading it.", ex, recSourceName, recSource, ex.Message); lock (recSourcesLockAndMemFence) { m_recSourceFactories.Remove(recSourceName); } } }, cancellationToken); recSourceTrainTasksList.Add(recSourceTrainTask); } // Wait for all to complete or cancellation. There should not be any exceptions other than OperationCanceledException. await Task.WhenAll(recSourceTrainTasksList).ConfigureAwait(false); lock (recSourcesLockAndMemFence) { ; // just for the fence } } if (finalize) { m_trainingData = null; m_usernames = null; m_finalized = true; Logging.Log.Info("Finalized rec sources."); } totalTimer.Stop(); Logging.Log.InfoFormat("All rec sources retrained with the latest data. Total time: {0}", totalTimer.Elapsed); } GC.Collect(); Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false)); }