private void AssertReadLockContended(AsyncUpgradeableReaderWriterLock rwLock)
{
using (CancellationTokenSource timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)))
{
Assert.Throws <TaskCanceledException>(() => rwLock.EnterReadLockAsync(timeout.Token).ConfigureAwait(false).GetAwaiter().GetResult());
}
}
private void AssertReadLockUncontended(AsyncUpgradeableReaderWriterLock rwLock)
{
using (CancellationTokenSource timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(200)))
// Will throw an OperationCanceledException if unable to get the lock within 200 ms
using (rwLock.EnterReadLockAsync(timeout.Token).ConfigureAwait(false).GetAwaiter().GetResult())
{
;
}
}
private Task HoldReadLockAsync(AsyncUpgradeableReaderWriterLock rwLock, ManualResetEventSlim setWhenLockHeld, CancellationToken cancellationToken)
{
return(Task.Run(() =>
{
using (rwLock.EnterReadLockAsync(CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult())
{
setWhenLockHeld.Set();
cancellationToken.WaitHandle.WaitOne();
}
}));
}
public async Task LoadRecSourceAsync(Func <ITrainableJsonRecSource> recSourceFactory, string name, bool replaceExisting, CancellationToken cancellationToken)
{
// Acquire read lock on current list, write lock on pending list
// If name already exists on current list and replaceExisting = false, throw.
// If name already exists on pending list, throw.
// Otherwise, add name to pending list, release locks, and proceed.
using (var recSourcesReadLock = await m_recSourcesLockAsync.EnterReadLockAsync(cancellationToken).ConfigureAwait(false))
using (var pendingRecSourcesWriteLock = await m_pendingRecSourcesLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false))
{
if (m_recSources.ContainsKey(name) && !replaceExisting)
{
throw new RecServiceErrorException(new Error(errorCode: ErrorCodes.Unknown,
message: string.Format("A recommendation source with the name \"{0}\" already exists.", name)));
}
if (m_pendingRecSources.Contains(name))
{
throw new RecServiceErrorException(new Error(errorCode: ErrorCodes.Unknown,
message: string.Format("A recommendation source with the name \"{0}\" is currently being trained.", name)));
}
m_pendingRecSources.Add(name);
}
try
{
// Need to hold read lock on training data while training so that a retrain can't happen while we're training here.
// Rec sources must be trained with the current m_trainingData, not an old version.
using (var trainingDataReadLock = await m_trainingDataLockAsync.EnterReadLockAsync(cancellationToken).ConfigureAwait(false))
{
if (m_trainingData == null && !m_finalized)
{
throw new RecServiceErrorException(new Error(errorCode: ErrorCodes.NoTrainingData,
message: "A reload/retrain in low memory mode failed, leaving the rec service without training data or rec sources. Issue a ReloadTrainingData command to load training data, then load rec sources."));
}
else if (m_trainingData == null && m_finalized)
{
throw new RecServiceErrorException(new Error(errorCode: ErrorCodes.Finalized,
message: "Rec sources have been finalized. A non-finalized retrain must be invoked to be able to add rec sources."));
}
ITrainableJsonRecSource recSource = recSourceFactory();
Logging.Log.InfoFormat("Training rec source with name \"{0}\", replaceExisting={1}: {2}", name, replaceExisting, recSource);
Stopwatch timer = Stopwatch.StartNew();
recSource.Train(m_trainingData, m_usernames, cancellationToken);
timer.Stop();
Logging.Log.InfoFormat("Trained rec source {0}. Took {1}", name, timer.Elapsed);
using (var recSourcesWriteLock = await m_recSourcesLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false))
using (var pendingRecSourcesWriteLock = await m_pendingRecSourcesLockAsync.EnterWriteLockAsync(cancellationToken).ConfigureAwait(false))
{
m_recSources[name] = recSource;
m_recSourceFactories[name] = recSourceFactory;
m_pendingRecSources.Remove(name);
}
}
}
catch (Exception)
{
m_pendingRecSources.Remove(name);
throw;
}
Logging.Log.InfoFormat("Loaded rec source {0}.", name);
GC.Collect();
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
}
