public static void FillClientSubmissionId(AntiPlagiarismDb adb) { var ids = adb.Submissions.Where(s => s.ClientSubmissionId == null).Select(s => s.Id).ToList(); adb.DisableAutoDetectChanges(); foreach (var id in ids) { var submission = adb.Submissions.Find(id); if (submission == null) { Console.WriteLine($"Stop on id {id}"); return; } try { submission.ClientSubmissionId = JsonConvert.DeserializeObject <AdditionalInfo>(submission.AdditionalInfo).SubmissionId.ToString(); adb.Submissions.Update(submission); } catch (Exception ex) { Console.WriteLine($"Error on id {id} \"{submission.AdditionalInfo}\""); } if (id % 1000 == 0) { adb.SaveChanges(); Console.WriteLine(id); } } adb.SaveChanges(); adb.EnableAutoDetectChanges(); }
/* We stands with perfomance issue on EF Core: https://github.com/aspnet/EntityFrameworkCore/issues/11680 * So we decided to disable AutoDetectChangesEnabled temporary for some queries */ private void DisableAutoDetectChanges() { logger.Debug("Выключаю AutoDetectChangesEnabled"); db.DisableAutoDetectChanges(); }
private static void FillLanguageSnippetsStatistics(AntiPlagiarismDb adb, bool @do) { Console.WriteLine("FillLanguageSnippetsStatistics"); var taskIds = adb .SnippetsStatistics .Select(s => s.TaskId) .Distinct() .ToList(); Console.WriteLine($"Count taskIds {taskIds.Count}"); var taskIdToLanguage = new Dictionary <Guid, Language?>(); foreach (var taskId in taskIds) { if (!taskIdToLanguage.ContainsKey(taskId)) { taskIdToLanguage[taskId] = GetLanguageByTaskId(taskId, adb); } } var snippets = adb .SnippetsStatistics .Select(c => new { c.Id, c.TaskId, c.Language }) .AsNoTracking(); var count = snippets.Count(); Console.WriteLine($"Count snippets {count}"); var changesCount = 0; var changes = new List <(int Id, Language Language)>(); using (var changesFile = new StreamWriter("changes.txt")) { var getChangesCompleted = 0; foreach (var snippet in snippets) { getChangesCompleted++; if (getChangesCompleted % 10000 == 0) { Console.WriteLine($"getChangesCompleted {getChangesCompleted} / {count}"); } var newLanguage = taskIdToLanguage[snippet.TaskId]; if (newLanguage == null) { newLanguage = 0; } if (newLanguage == snippet.Language) { continue; } changesFile.WriteLine($"{snippet.Id} {newLanguage.Value}"); changes.Add((snippet.Id, newLanguage.Value)); changesCount++; } } Console.WriteLine($"Found changes {changesCount}"); var batchSize = 300; var currentBatch = 0; var batches = new List <List <(int Id, Language Language)> >(); while (currentBatch * batchSize < changes.Count) { var nextBatch = changes.Skip(currentBatch * batchSize) .Take(batchSize).ToList(); batches.Add(nextBatch); currentBatch++; } if (@do) { adb.DisableAutoDetectChanges(); var completed = 0; foreach (var batch in batches) { var batchIds = batch.Select(s => s.Id).ToList(); var batchDict = batch.ToDictionary(s => s.Id, s => s.Language); var batchData = adb.SnippetsStatistics.Where(s => batchIds.Contains(s.Id)).ToList(); foreach (var data in batchData) { data.Language = batchDict[data.Id]; adb.SnippetsStatistics.Update(data); } completed += batch.Count; Console.WriteLine($"FillLanguageSnippetsStatistics - Completed {completed} / {changesCount}"); adb.SaveChanges(); } adb.EnableAutoDetectChanges(); } }