public void SynchronouslyCanceledTasksDoNotTakeOverBookkeeping()
{
// Tests fix for a bug where, when a task faults and triggers cancellation of all tasks, the
// continuation for some canceled tasks (such as those not started yet) runs synchronously on the
// same thread. Since the canceled task's continuation runs on the same thread as the faulted task's
// continuation, a lock statement is not effective for mutual exclusion. The canceled task's
// continuation runs through all the bookkeeping and ultimately results in the aggregate task
// ending in a canceled state instead of a faulted state.
// To recreate this situation, we need a task that faults and a task associated with a cancellation token
// that does not start executing until the first task has had time to fault and run its continuation,
// and a third task that is in the running state when the first task runs its continuation.
CancellationTokenSource secondTaskCts = new CancellationTokenSource();
Task delayStartTask = new Task(() =>
{
secondTaskCts.Token.ThrowIfCancellationRequested();
}, secondTaskCts.Token);
CancellationTokenSource thirdTaskCts = new CancellationTokenSource();
Task runningTask = Task.Run(() =>
{
thirdTaskCts.Token.ThrowIfCancellationRequested();
Thread.Sleep(TimeSpan.FromMilliseconds(2000));
thirdTaskCts.Token.ThrowIfCancellationRequested();
}, thirdTaskCts.Token);
CancellableTask[] tasks = new CancellableTask[]
{
GetLateFaultingCancellableTask(),
new CancellableTask(delayStartTask, secondTaskCts),
new CancellableTask(runningTask, thirdTaskCts),
};
Task.Delay(TimeSpan.FromMilliseconds(2000)).ContinueWith(task =>
{
delayStartTask.Start();
});
try
{
Task waitTask = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(tasks);
Assert.Throws <Exception>(() => waitTask.ConfigureAwait(false).GetAwaiter().GetResult());
Assert.Single(waitTask.Exception.InnerExceptions);
Assert.All(waitTask.Exception.InnerExceptions, ex => Assert.Equal("Late Fault", ex.Message));
Assert.Equal(TaskStatus.Faulted, tasks[0].Task.Status);
Assert.Equal(TaskStatus.Canceled, tasks[1].Task.Status);
TaskStatus task3Status = tasks[2].Task.Status;
Assert.True(task3Status == TaskStatus.Canceled || task3Status == TaskStatus.Running, string.Format("Third task has status {0} instead of Canceled or Running.", task3Status));
}
finally
{
foreach (CancellableTask task in tasks)
{
task.CancellationTokenSource.Dispose();
}
}
}
public void CancelBeforeOthersFaultAfterCancellationDontWaitForCancellation()
{
// 1 task canceled, others basically wait for cancellation and wait a bit after receiving cancellation,
// 2 others fault after cancellation, cancel occurs before successes and faults, waitForCancellation false
// -> TaskCanceledException with Task == canceled task, other tasks are not completed yet
CancellableTask[] tasks = new CancellableTask[]
{
GetEarlyCancellingCancellableTask(),
GetLateFaultingCancellableTask(),
GetLateFaultingCancellableTask(),
GetSuccessfulCancellableTask(),
GetSuccessfulCancellableTask()
};
try
{
Task waitTask = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(tasks);
bool taskCanceledCaught = false;
try
{
waitTask.ConfigureAwait(false).GetAwaiter().GetResult();
}
catch (TaskCanceledException ex)
{
taskCanceledCaught = true;
ex.CancellationToken.Should().Be(tasks[0].CancellationTokenSource.Token);
}
TaskStatus task1Status = tasks[1].Task.Status;
TaskStatus task2Status = tasks[2].Task.Status;
TaskStatus task3Status = tasks[3].Task.Status;
TaskStatus task4Status = tasks[4].Task.Status;
Assert.True(taskCanceledCaught, "TaskCanceledException was not thrown.");
Assert.Equal(TaskStatus.Canceled, tasks[0].Task.Status);
Assert.True(task1Status == TaskStatus.Running || task1Status == TaskStatus.WaitingForActivation, string.Format("Second task has status {0} instead of Running or WaitingForActivation.", task1Status));
Assert.True(task2Status == TaskStatus.Running || task2Status == TaskStatus.WaitingForActivation, string.Format("Third task has status {0} instead of Running or WaitingForActivation.", task2Status));
Assert.True(task3Status == TaskStatus.Running || task3Status == TaskStatus.WaitingForActivation, string.Format("Fourth task has status {0} instead of Running or WaitingForActivation.", task3Status));
Assert.True(task4Status == TaskStatus.Running || task4Status == TaskStatus.WaitingForActivation, string.Format("Fifth task has status {0} instead of Running or WaitingForActivation.", task4Status));
}
finally
{
foreach (CancellableTask task in tasks)
{
task.CancellationTokenSource.Dispose();
}
}
}
public void FaultsBeforeSuccessesDontWaitForCancellation()
{
// 2 tasks fault, others basically wait for cancellation and wait a bit after receiving cancellation,
// fault occurs before successes, waitForCancellations false
// -> AggregateException with at least 1 fault inside, at least one of first two tasks in faulted state, other tasks are not completed yet
CancellableTask[] tasks = new CancellableTask[]
{
GetEarlyFaultingCancellableTask(),
GetEarlyFaultingCancellableTask(),
GetSuccessfulCancellableTask(),
GetSuccessfulCancellableTask()
};
try
{
Task waitTask = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(tasks);
Assert.Throws <Exception>(() => waitTask.ConfigureAwait(false).GetAwaiter().GetResult());
Assert.Equal(1, waitTask.Exception.InnerExceptions.Count);
Assert.Equal("Early Fault", waitTask.Exception.InnerException.Message);
TaskStatus task0Status = tasks[0].Task.Status;
TaskStatus task1Status = tasks[1].Task.Status;
TaskStatus task2Status = tasks[2].Task.Status;
TaskStatus task3Status = tasks[3].Task.Status;
int numFaulted = (task0Status == TaskStatus.Faulted ? 1 : 0) + (task1Status == TaskStatus.Faulted ? 1 : 0);
numFaulted.Should().BeGreaterOrEqualTo(1);
Assert.True(task0Status == TaskStatus.Faulted || task0Status == TaskStatus.Running || task0Status == TaskStatus.WaitingForActivation, string.Format("First task has status {0} instead of Faulted, Running, or WaitingForActivation.", task0Status));
Assert.True(task1Status == TaskStatus.Faulted || task1Status == TaskStatus.Running || task1Status == TaskStatus.WaitingForActivation, string.Format("Second task has status {0} instead of Faulted, Running, or WaitingForActivation.", task1Status));
Assert.True(task2Status == TaskStatus.Running || task2Status == TaskStatus.WaitingForActivation, string.Format("Third task has status {0} instead of Running or WaitingForActivation.", task2Status));
Assert.True(task3Status == TaskStatus.Running || task3Status == TaskStatus.WaitingForActivation, string.Format("Fourth task has status {0} instead of Running or WaitingForActivation.", task3Status));
}
finally
{
foreach (CancellableTask task in tasks)
{
task.CancellationTokenSource.Dispose();
}
}
}
public void CancelAfterSuccessesDontWaitForCancellation()
{
// 1 task canceled, others succeed, cancel occurs after successes, waitForCancellations false
// -> TaskCanceledException with CancellationToken == canceled task's cancellation token
CancellableTask[] tasks = new CancellableTask[]
{
GetLateCancellingCancellableTask(),
GetSuccessfulCancellableTask(),
GetSuccessfulCancellableTask()
};
try
{
Task waitTask = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(tasks);
bool taskCanceledCaught = false;
try
{
waitTask.ConfigureAwait(false).GetAwaiter().GetResult();
}
catch (TaskCanceledException ex)
{
taskCanceledCaught = true;
ex.CancellationToken.Should().Be(tasks[0].CancellationTokenSource.Token);
}
Assert.True(taskCanceledCaught, "TaskCanceledException was not thrown.");
Assert.Equal(TaskStatus.Canceled, tasks[0].Task.Status);
Assert.Equal(TaskStatus.RanToCompletion, tasks[1].Task.Status);
Assert.Equal(TaskStatus.RanToCompletion, tasks[2].Task.Status);
}
finally
{
foreach (CancellableTask task in tasks)
{
task.CancellationTokenSource.Dispose();
}
}
}
// Loads all configured rec sources into the rec service in parallel.
// Does not return until complete or serviceStopToken is signaled.
private static void LoadRecSources(TcpRecService recService, Config config, CancellationToken serviceStopToken)
{
if (config.RecSources.Count == 0)
{
Logging.Log.Info("No rec sources configured.");
return;
}
Logging.Log.InfoFormat("Loading {0} rec sources.", config.RecSources.Count);
List <ICancellableTask> recSourceLoadTasks = new List <ICancellableTask>(config.RecSources.Count);
using (CancellationTokenSource anyTaskFaultedOrCanceled = new CancellationTokenSource())
using (CancellationTokenSource cancelTokenSource = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, anyTaskFaultedOrCanceled.Token))
{
foreach (DTO.LoadRecSourceRequest recSourceConfigX in config.RecSources)
{
DTO.LoadRecSourceRequest recSourceConfig = recSourceConfigX; // Don't capture the loop variable
Task loadRecSourceTask = Task.Factory.StartNew(() =>
{
recService.LoadRecSource(recSourceConfig, cancelTokenSource.Token);
}, cancelTokenSource.Token);
recSourceLoadTasks.Add(new CancellableTask(loadRecSourceTask, anyTaskFaultedOrCanceled));
}
try
{
AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(recSourceLoadTasks).ConfigureAwait(false).GetAwaiter().GetResult();
}
catch (OperationCanceledException)
{
Logging.Log.Info("Canceled loading rec sources.");
throw;
}
}
recService.FinalizeRecSources(serviceStopToken);
}
public void AllCompleteSuccessfullyDontWaitForCancellations()
{
CancellableTask[] tasks = new CancellableTask[3]
{
GetSuccessfulCancellableTask(),
GetSuccessfulCancellableTask(),
GetSuccessfulCancellableTask()
};
try
{
Task waitTask = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(tasks);
waitTask.ConfigureAwait(false).GetAwaiter().GetResult();
Assert.All(tasks, task => Assert.Equal(TaskStatus.RanToCompletion, task.Task.Status));
}
finally
{
foreach (CancellableTask task in tasks)
{
task.CancellationTokenSource.Dispose();
}
}
}
// Loads training data and prerequisites from the database in parallel and does not return until they are loaded.
private static (MalTrainingData trainingData, IDictionary <int, IList <int> > prereqs) LoadInitialData(IMalTrainingDataLoaderFactory trainingDataLoaderFactory, CancellationToken serviceStopToken)
{
using (IMalTrainingDataLoader initialTrainingDataLoader = trainingDataLoaderFactory.GetTrainingDataLoader())
using (CancellationTokenSource trainingDataOtherFaultOrCancellation = new CancellationTokenSource())
using (CancellationTokenSource trainingDataCancel = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, trainingDataOtherFaultOrCancellation.Token))
using (CancellationTokenSource prereqsOtherFaultOrCancellation = new CancellationTokenSource())
using (CancellationTokenSource prereqsCancel = CancellationTokenSource.CreateLinkedTokenSource(serviceStopToken, prereqsOtherFaultOrCancellation.Token))
{
CancellableAsyncFunc <MalTrainingData> trainingDataAsyncFunc = new CancellableAsyncFunc <MalTrainingData>(
() => LoadTrainingDataOnInitAsync(initialTrainingDataLoader, trainingDataCancel.Token), trainingDataOtherFaultOrCancellation);
CancellableTask <MalTrainingData> trainingDataTask = trainingDataAsyncFunc.StartTaskEnsureExceptionsWrapped();
CancellableAsyncFunc <IDictionary <int, IList <int> > > prereqsAsyncFunc = new CancellableAsyncFunc <IDictionary <int, IList <int> > >(
() => LoadPrereqsOnInit(initialTrainingDataLoader, prereqsCancel.Token), prereqsOtherFaultOrCancellation);
CancellableTask <IDictionary <int, IList <int> > > prereqsTask = prereqsAsyncFunc.StartTaskEnsureExceptionsWrapped();
AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(trainingDataTask, prereqsTask).ConfigureAwait(false).GetAwaiter().GetResult();
return(trainingDataTask.Task.Result, prereqsTask.Task.Result);
}
}
public void FaultsAfterSuccessesDontWaitForCancellations()
{
// 2 tasks fault, others succeed, faults occur after successes, waitForCancellations false
// -> AggregateException with at least one fault inside
CancellableTask[] tasks = new CancellableTask[]
{
GetLateFaultingCancellableTask(),
GetLateFaultingCancellableTask(),
GetSuccessfulCancellableTask(),
GetSuccessfulCancellableTask()
};
try
{
Task waitTask = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(tasks);
Assert.Throws <Exception>(() => waitTask.ConfigureAwait(false).GetAwaiter().GetResult());
Assert.Equal(1, waitTask.Exception.InnerExceptions.Count);
Assert.Equal("Late Fault", waitTask.Exception.InnerException.Message);
TaskStatus task0Status = tasks[0].Task.Status;
TaskStatus task1Status = tasks[1].Task.Status;
int numFaulted = (task0Status == TaskStatus.Faulted ? 1 : 0) + (task1Status == TaskStatus.Faulted ? 1 : 0);
numFaulted.Should().BeGreaterOrEqualTo(1);
Assert.True(task0Status == TaskStatus.Faulted || task0Status == TaskStatus.Running || task0Status == TaskStatus.WaitingForActivation, string.Format("First task has status {0} instead of Faulted, Running, or WaitingForActivation.", task0Status));
Assert.True(task1Status == TaskStatus.Faulted || task1Status == TaskStatus.Running || task1Status == TaskStatus.WaitingForActivation, string.Format("Second task has status {0} instead of Faulted, Running, or WaitingForActivation.", task1Status));
Assert.Equal(TaskStatus.RanToCompletion, tasks[2].Task.Status);
Assert.Equal(TaskStatus.RanToCompletion, tasks[3].Task.Status);
}
finally
{
foreach (CancellableTask task in tasks)
{
task.CancellationTokenSource.Dispose();
}
}
}
public void PassingNoTasksReturnsInstantlyDontWaitForCancellations()
{
Task task = AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(new CancellableTask[0]);
Assert.Equal(TaskStatus.RanToCompletion, task.Status);
}
public async Task <MalTrainingData> LoadMalTrainingDataAsync(CancellationToken cancellationToken)
{
// Load all anime, users, and entries in parallel, then combine them into training data.
try
{
Dictionary <int, MalAnime> animes;
Dictionary <int, mal_user> dbUsers;
IList <mal_list_entry_slim> dbEntrySlurp;
using (CancellationTokenSource faultCanceler = new CancellationTokenSource())
using (CancellationTokenSource faultOrUserCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, faultCanceler.Token))
{
Task <Dictionary <int, MalAnime> > animeTask = AsyncUtils.EnsureExceptionsWrapped(
() => SlurpAnimeAsync(faultOrUserCancel.Token));
CancellableTask cancellableAnimeTask = new CancellableTask(animeTask, faultCanceler);
Task <Dictionary <int, mal_user> > userTask = AsyncUtils.EnsureExceptionsWrapped(
() => SlurpUsersAsync(faultOrUserCancel.Token));
CancellableTask cancellableUserTask = new CancellableTask(userTask, faultCanceler);
Task <IList <mal_list_entry_slim> > entryTask = AsyncUtils.EnsureExceptionsWrapped(
() => SlurpEntriesAsync(faultOrUserCancel.Token));
CancellableTask cancellableEntryTask = new CancellableTask(entryTask, faultCanceler);
await AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(cancellableEntryTask, cancellableAnimeTask, cancellableUserTask).ConfigureAwait(false);
animes = animeTask.Result;
dbUsers = userTask.Result;
dbEntrySlurp = entryTask.Result;
}
Logging.Log.Debug("Processing list entries from the database.");
long entryCount = 0;
Dictionary <int, List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> > entriesByUser =
new Dictionary <int, List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> >();
foreach (mal_list_entry_slim dbEntry in dbEntrySlurp)
{
entryCount++;
mal_user dbUser;
if (!dbUsers.TryGetValue(dbEntry.mal_user_id, out dbUser) || !animes.ContainsKey(dbEntry.mal_anime_id))
{
// Entry for an anime or user that wasn't in the database...there must have been an update going on between the time we got users, anime, and list entries
continue;
}
List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> animeList;
if (!entriesByUser.TryGetValue(dbEntry.mal_user_id, out animeList))
{
animeList = new List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId>();
entriesByUser[dbEntry.mal_user_id] = animeList;
}
animeList.Add(new ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId(
animeId: dbEntry.mal_anime_id,
entry: new MalListEntry(
rating: (byte?)dbEntry.rating,
status: (CompletionStatus)dbEntry.mal_list_entry_status_id,
numEpisodesWatched: dbEntry.num_episodes_watched
)
));
}
Dictionary <int, MalUserListEntries> users = new Dictionary <int, MalUserListEntries>(dbUsers.Count);
foreach (int userId in entriesByUser.Keys)
{
List <ReadOnlyMalListEntryDictionary.ListEntryAndAnimeId> animeList = entriesByUser[userId];
animeList.Capacity = animeList.Count;
ReadOnlyMalListEntryDictionary listEntries = new ReadOnlyMalListEntryDictionary(animeList);
users[userId] = new MalUserListEntries(listEntries, animes, dbUsers[userId].mal_name, okToRecommendPredicate: null);
}
Logging.Log.DebugFormat("Done processing {0} list entries.", entryCount);
return(new MalTrainingData(users, animes));
}
catch (Exception ex) when(!(ex is OperationCanceledException))
{
throw new Exception(string.Format("Error loading MAL training data: {0}", ex.Message), ex);
}
}
/// <summary>
/// If null inputfile, create fresh.
/// </summary>
/// <param name="inputFile"></param>
/// <param name="outputFile"></param>
static void CreateCsv(CommandLineArgs args)
{
string inputFile = args.InputFile;
string outputFile = args.OutputFile;
// Read in existing mapping from the input csv. If no input csv was specified, treat it as an empty csv.
List <CsvRow> inputCsvRows;
if (inputFile == null)
{
inputCsvRows = new List <CsvRow>();
}
else
{
inputCsvRows = LoadCsv(inputFile);
}
// Index streams available by the streaming service and the anime name used by the streaming service.
Dictionary <StreamingService, Dictionary <string, List <CsvRow> > > rowsByServiceAndAnime = new Dictionary <StreamingService, Dictionary <string, List <CsvRow> > >();
foreach (CsvRow csvRow in inputCsvRows)
{
if (!rowsByServiceAndAnime.ContainsKey(csvRow.Service))
{
rowsByServiceAndAnime[csvRow.Service] = new Dictionary <string, List <CsvRow> >();
}
if (!rowsByServiceAndAnime[csvRow.Service].ContainsKey(csvRow.AnimeName))
{
rowsByServiceAndAnime[csvRow.Service][csvRow.AnimeName] = new List <CsvRow>();
}
rowsByServiceAndAnime[csvRow.Service][csvRow.AnimeName].Add(csvRow);
}
List <AnimeStreamInfo> streams = new List <AnimeStreamInfo>();
using (WebClient webClient = new WebClient())
// Use Firefox driver with headless Firefox for Funimation to get around Incapsula.
// Use Firefox and not Chrome because javascript cannot be disabled with headless Chrome at this time.
// Javascript slows the process down and is not needed currently.
using (FirefoxDriverWebClient funimationWebClient = new FirefoxDriverWebClient(args.GeckoDriverDirectory))
{
List <IAnimeStreamInfoSource> streamInfoSources = GetStreamInfoSources(args, webClient, funimationWebClient);
using (CancellationTokenSource cancellation = new CancellationTokenSource(TimeSpan.FromMinutes(5)))
{
CancellableAsyncFunc <ICollection <AnimeStreamInfo> >[] streamFuncs = streamInfoSources.Select(source => new CancellableAsyncFunc <ICollection <AnimeStreamInfo> >(
() => source.GetAnimeStreamInfoAsync(cancellation.Token), cancellation)
).ToArray();
CancellableTask <ICollection <AnimeStreamInfo> >[] streamTasks = AsyncUtils.StartTasksEnsureExceptionsWrapped(streamFuncs);
try
{
AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(streamTasks).GetAwaiter().GetResult();
}
catch (OperationCanceledException ex)
{
throw new Exception("Getting streams timed out.", ex);
}
foreach (CancellableTask <ICollection <AnimeStreamInfo> > streamTask in streamTasks)
{
streams.AddRange(streamTask.Task.Result);
}
}
}
Dictionary <StreamingService, Dictionary <string, List <AnimeStreamInfo> > > streamsByServiceAndAnime = new Dictionary <StreamingService, Dictionary <string, List <AnimeStreamInfo> > >();
foreach (AnimeStreamInfo stream in streams)
{
if (!streamsByServiceAndAnime.ContainsKey(stream.Service))
{
streamsByServiceAndAnime[stream.Service] = new Dictionary <string, List <AnimeStreamInfo> >();
}
if (!streamsByServiceAndAnime[stream.Service].ContainsKey(stream.AnimeName))
{
streamsByServiceAndAnime[stream.Service][stream.AnimeName] = new List <AnimeStreamInfo>();
}
streamsByServiceAndAnime[stream.Service][stream.AnimeName].Add(stream);
}
Console.WriteLine("Writing out csv.");
// Write a new csv mapping to the output file. If MAL anime ids or n/a was present in the input file for a certain
// streaming service/anime name/URL combination, use them. Otherwise, leave the MAL anime id column blank
// for a human operator to fill in.
using (FileStream outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read))
using (StreamWriter output = new StreamWriter(outputStream, Encoding.UTF8))
{
string header = "Service,Anime,URL,MAL ID (or n/a)";
output.Write(header); // Newline gets written by the first row's data
foreach (AnimeStreamInfo streamInfo in streams
.OrderBy(stream => stream.Service.ToString())
.ThenBy(stream => stream.AnimeName)
.ThenBy(stream => stream.Url))
{
List <CsvRow> existingCsvRows = new List <CsvRow>();
if (rowsByServiceAndAnime.ContainsKey(streamInfo.Service) && rowsByServiceAndAnime[streamInfo.Service].ContainsKey(streamInfo.AnimeName))
{
List <CsvRow> rowsForThisServiceAndAnime = rowsByServiceAndAnime[streamInfo.Service][streamInfo.AnimeName];
// Amazon URLs look like https://www.amazon.com/Our-Eyes-Finally-Met-Anothers/dp/B06Y5WC21S
// The "Our-Eyes-Finally-Met-Anothers" relates to an episode title.
// The B06Y5WC21S is some sort of ID that seems to relate to the episode rather than the whole series.
// For reasons unknown, the episode that represents the whole series can change, resulting in the URL changing.
// This results in a fair amount of churn in the CSV, around 20 changes per week.
// To avoid having to remap those streams to MAL IDs, use the following logic:
// If amazon service, and only one URL present in existing CSV for this (service, title),
// and only one url present in the streams that we just got, then consider the existing CSV rows
// a match and use their MAL IDs.
// Even if that is not the case, then only consider the ID at the end when matching URLs of streams
// to existing CSV rows.
if (streamInfo.Service != StreamingService.AmazonPrime)
{
existingCsvRows = rowsForThisServiceAndAnime.Where(row => row.Url == streamInfo.Url).ToList();
}
else
{
if (rowsForThisServiceAndAnime.GroupBy(row => row.Url).Count() == 1 && streamsByServiceAndAnime[streamInfo.Service][streamInfo.AnimeName].GroupBy(stream => stream.Url).Count() == 1)
{
existingCsvRows = rowsForThisServiceAndAnime.ToList();
}
else
{
string amazonStreamID = GetAmazonIDFromUrl(streamInfo.Url);
existingCsvRows = rowsForThisServiceAndAnime.Where(row => GetAmazonIDFromUrl(row.Url) == amazonStreamID).ToList();
}
}
}
if (existingCsvRows.Count > 0)
{
foreach (CsvRow existingRow in existingCsvRows)
{
// not WriteLine() - this should be \r\n regardless of what platform this is run on per the CSV RFC
// Header row did not write a newline, so there won't be a blank line between header and first row.
output.Write("\r\n");
output.Write("{0},{1},{2},{3}", QuoteForCsv(streamInfo.Service.ToString()),
QuoteForCsv(streamInfo.AnimeName), QuoteForCsv(streamInfo.Url),
existingRow.MalAnimeId.ToString());
}
}
else
{
output.Write("\r\n");
// Notice the comma at the end - leave MAL anime id blank for a human operator to fill in.
output.Write("{0},{1},{2},", QuoteForCsv(streamInfo.Service.ToString()), QuoteForCsv(streamInfo.AnimeName),
QuoteForCsv(streamInfo.Url));
}
}
}
}
private async Task ReloadTrainingDataHighAvailabilityAsync(bool finalize, CancellationToken cancellationToken)
{
using (var trainingDataUpgradeableLock = await m_trainingDataLockAsync.EnterUpgradeableReadLockAsync(cancellationToken).ConfigureAwait(false))
{
Logging.Log.Info("Reloading training data and retraining rec sources. Rec sources will remain available.");
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
Stopwatch totalTimer = Stopwatch.StartNew();
// Load new training data
MalTrainingData newData;
IDictionary <int, string> newUsernames;
IDictionary <int, IList <int> > newPrereqs;
using (IMalTrainingDataLoader malTrainingDataLoader = m_trainingDataLoaderFactory.GetTrainingDataLoader())
using (CancellationTokenSource faultCanceler = new CancellationTokenSource())
using (CancellationTokenSource faultOrUserCancel = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, faultCanceler.Token))
{
Stopwatch trainingDataTimer = Stopwatch.StartNew();
CancellableTask <MalTrainingData> trainingDataTask = new CancellableTask <MalTrainingData>(
malTrainingDataLoader.LoadMalTrainingDataAsync(faultOrUserCancel.Token), faultCanceler);
Task trainingDataTimerTask = trainingDataTask.Task.ContinueWith(task =>
{
trainingDataTimer.Stop();
Logging.Log.InfoFormat("Training data loaded. {0} users, {1} animes, {2} entries. Took {3}.",
task.Result.Users.Count, task.Result.Animes.Count,
task.Result.Users.Keys.Sum(userId => task.Result.Users[userId].Entries.Count),
trainingDataTimer.Elapsed);
},
cancellationToken, TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnCanceled
| TaskContinuationOptions.NotOnFaulted, TaskScheduler.Current);
Stopwatch prereqsTimer = Stopwatch.StartNew();
CancellableTask <IDictionary <int, IList <int> > > prereqsTask = new CancellableTask <IDictionary <int, IList <int> > >(
malTrainingDataLoader.LoadPrerequisitesAsync(faultOrUserCancel.Token), faultCanceler);
Task prereqsTimerTask = prereqsTask.Task.ContinueWith(task =>
{
prereqsTimer.Stop();
int numPrereqs = task.Result.Values.Sum(prereqList => prereqList.Count);
Logging.Log.InfoFormat("Prerequisites loaded. {0} prerequisites for {1} animes. Took {2}.",
numPrereqs, task.Result.Count, prereqsTimer.Elapsed);
},
cancellationToken, TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnCanceled
| TaskContinuationOptions.NotOnFaulted, TaskScheduler.Current);
await AsyncUtils.WhenAllCancelOnFirstExceptionDontWaitForCancellations(trainingDataTask, prereqsTask);
newData = trainingDataTask.Task.Result;
newUsernames = GetUsernamesFromTrainingData(newData);
newPrereqs = prereqsTask.Task.Result;
await trainingDataTimerTask.ConfigureAwait(false);
await prereqsTimerTask.ConfigureAwait(false);
}
GC.Collect();
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
using (var recSourcesUpgradeableLock = await m_recSourcesLockAsync.EnterUpgradeableReadLockAsync(cancellationToken).ConfigureAwait(false))
{
// clone the json rec sources without the training state and train each one with the new data.
Dictionary <string, ITrainableJsonRecSource> newRecSources = new Dictionary <string, ITrainableJsonRecSource>(StringComparer.OrdinalIgnoreCase);
Dictionary <string, Func <ITrainableJsonRecSource> > newRecSourceFactories = new Dictionary <string, Func <ITrainableJsonRecSource> >(m_recSourceFactories, StringComparer.OrdinalIgnoreCase);
if (m_recSourceFactories.Count == 0)
{
Logging.Log.Info("No rec sources to retrain.");
}
else
{
Logging.Log.Info("Retraining rec sources.");
object newRecSourcesLockAndMemFence = new object();
List <Task> recSourceTrainTasksList = new List <Task>();
// ToList() so we can unload a rec source as we iterate if it errors while training.
foreach (string recSourceNameLoopVar in m_recSourceFactories.Keys.ToList())
{
string recSourceName = recSourceNameLoopVar; // avoid capturing the loop var
ITrainableJsonRecSource recSource = newRecSourceFactories[recSourceName]();
Task recSourceTrainTask = Task.Run(() =>
{
Logging.Log.InfoFormat("Retraining rec source {0} ({1}).", recSourceName, recSource);
Stopwatch trainTimer = Stopwatch.StartNew();
try
{
recSource.Train(newData, newUsernames, cancellationToken);
trainTimer.Stop();
Logging.Log.InfoFormat("Trained rec source {0} ({1}). Took {2}.", recSourceName, recSource, trainTimer.Elapsed);
lock (newRecSourcesLockAndMemFence)
{
newRecSources[recSourceName] = recSource;
}
}
catch (OperationCanceledException)
{
Logging.Log.InfoFormat("Canceled while retraining rec source {0} ({1}).", recSourceName, recSource);
throw;
}
catch (Exception ex)
{
Logging.Log.ErrorFormat("Error retraining rec source {0} ({1}): {2} Unloading it.",
ex, recSourceName, recSource, ex.Message);
lock (newRecSourcesLockAndMemFence)
{
newRecSourceFactories.Remove(recSourceName);
}
}
}, cancellationToken);
recSourceTrainTasksList.Add(recSourceTrainTask);
}
// Wait for all to complete or cancellation. There should not be any exceptions other than OperationCanceledException.
await Task.WhenAll(recSourceTrainTasksList);
lock (newRecSourcesLockAndMemFence)
{
; // just for the fence
}
}
// Swap in the newly trained rec sources.
using (var trainingDataWriteLock = await m_trainingDataLockAsync.UpgradeToWriteLock(cancellationToken).ConfigureAwait(false))
using (var recSourcesWriteLock = await m_recSourcesLockAsync.UpgradeToWriteLock(cancellationToken).ConfigureAwait(false))
{
m_recSources = newRecSources;
m_recSourceFactories = newRecSourceFactories;
m_animes = newData.Animes;
m_prereqs = newPrereqs;
if (finalize)
{
m_trainingData = null;
m_usernames = null;
m_finalized = true;
Logging.Log.Info("Finalized rec sources.");
}
else
{
m_trainingData = newData;
m_usernames = newUsernames;
m_finalized = false;
}
}
}
totalTimer.Stop();
Logging.Log.InfoFormat("All rec sources retrained with the latest data. Total time: {0}", totalTimer.Elapsed);
}
GC.Collect();
Logging.Log.InfoFormat("Memory use: {0} bytes", GC.GetTotalMemory(forceFullCollection: false));
}
