public static void FillClientSubmissionId(AntiPlagiarismDb adb)
{
var ids = adb.Submissions.Where(s => s.ClientSubmissionId == null).Select(s => s.Id).ToList();
adb.DisableAutoDetectChanges();
foreach (var id in ids)
{
var submission = adb.Submissions.Find(id);
if (submission == null)
{
Console.WriteLine($"Stop on id {id}");
return;
}
try
{
submission.ClientSubmissionId = JsonConvert.DeserializeObject <AdditionalInfo>(submission.AdditionalInfo).SubmissionId.ToString();
adb.Submissions.Update(submission);
}
catch (Exception ex)
{
Console.WriteLine($"Error on id {id} \"{submission.AdditionalInfo}\"");
}
if (id % 1000 == 0)
{
adb.SaveChanges();
Console.WriteLine(id);
}
}
adb.SaveChanges();
adb.EnableAutoDetectChanges();
}
public static async Task Main(string[] args)
{
var configuration = ApplicationConfiguration.Read <UlearnConfiguration>();
LoggerSetup.Setup(configuration.HostLog, configuration.GraphiteServiceName);
var optionsBuilder = new DbContextOptionsBuilder <UlearnDb>()
.UseLazyLoadingProxies()
.UseSqlServer(configuration.Database);
var db = new UlearnDb(optionsBuilder.Options);
var aOptionsBuilder = new DbContextOptionsBuilder <AntiPlagiarismDb>()
.UseLazyLoadingProxies()
.UseSqlServer(configuration.Database);
var adb = new AntiPlagiarismDb(aOptionsBuilder.Options);
//await ResendLti(db);
//await FindExternalSolutionsPlagiarism.UploadSolutions();
//await FindExternalSolutionsPlagiarism.GetRawResults();
//await FindExternalSolutionsPlagiarism.PrepareResults();
//await UpdateExerciseVisits(db, "fpIntroduction");
//Users.PrintCourseAdmins(db);
//await ScoresUpdater.UpdateTests(db, "java-rtf");
//GetMostSimilarSubmission(adb);
//ParsePairWeightsFromLogs();
//GetBlackAndWhiteLabels(db, adb);
//ParseTaskWeightsFromLogs();
//CampusRegistration();
//GetIps(db);
//FillAntiplagFields.FillClientSubmissionId(adb);
//await XQueueRunAutomaticChecking(db);
}
private static void GetMostSimilarSubmission(AntiPlagiarismDb adb)
{
//var lines = File.ReadLines("pairweights.txt");
//var jsons = AntiplagiarismLogsParser.GetWeightsOfSubmissionPairs(lines).Select(JsonConvert.SerializeObject);
//File.WriteAllLines("result.txt", jsons);
var bestPairWeight = File.ReadLines("result.txt").Select(JsonConvert.DeserializeObject <BestPairWeight>);
var now = DateTime.UtcNow;
var mostSimilarSubmissions = bestPairWeight.Select(s => new MostSimilarSubmission
{
SubmissionId = s.Submission,
SimilarSubmissionId = s.Other,
Weight = s.Weight,
Timestamp = now
}).ToList();
var exist = adb.MostSimilarSubmissions.Select(s => s.SubmissionId).ToList().ToHashSet();
var i = 0;
foreach (var mostSimilarSubmission in mostSimilarSubmissions)
{
if (exist.Contains(mostSimilarSubmission.SubmissionId))
{
continue;
}
adb.MostSimilarSubmissions.Add(mostSimilarSubmission);
if (i % 1000 == 0)
{
adb.SaveChanges();
Console.WriteLine(i);
}
i++;
}
adb.SaveChanges();
}
private static void FillLanguageTasksStatisticsParameters(AntiPlagiarismDb adb)
{
Console.WriteLine("FillLanguageTasksStatisticsParameters");
var parameterses = adb
.TasksStatisticsParameters
.Where(p => p.Language == 0)
.ToList();
var count = parameterses.Count;
Console.WriteLine($"Count {count}");
foreach (var parameters in parameterses)
{
var newLanguage = GetLanguageByTaskId(parameters.TaskId, adb);
if (newLanguage == null)
{
continue;
}
adb.TasksStatisticsParameters.Remove(parameters);
adb.SaveChanges();
parameters.Language = newLanguage.Value;
adb.TasksStatisticsParameters.Add(parameters);
adb.SaveChanges();
}
}
private static void FillLanguageManualSuspicionLevels(AntiPlagiarismDb adb)
{
Console.WriteLine("FillLanguageManualSuspicionLevels");
var suspicionLevels = adb
.ManualSuspicionLevels
.Where(p => p.Language == 0)
.ToList();
var count = suspicionLevels.Count;
Console.WriteLine($"Count {count}");
foreach (var level in suspicionLevels)
{
var newLanguage = GetLanguageByTaskId(level.TaskId, adb);
if (newLanguage == null)
{
continue;
}
adb.ManualSuspicionLevels.Remove(level);
adb.SaveChanges();
level.Language = newLanguage.Value;
adb.ManualSuspicionLevels.Add(level);
adb.SaveChanges();
}
}
public ApiController(
AntiPlagiarismDb db,
ISubmissionsRepo submissionsRepo, ISnippetsRepo snippetsRepo, ITasksRepo tasksRepo,
IClientsRepo clientsRepo, IWorkQueueRepo workQueueRepo,
IMostSimilarSubmissionsRepo mostSimilarSubmissionsRepo,
IManualSuspicionLevelsRepo manualSuspicionLevelsRepo,
PlagiarismDetector plagiarismDetector,
CodeUnitsExtractor codeUnitsExtractor,
IServiceScopeFactory serviceScopeFactory,
NewSubmissionHandler newSubmissionHandler,
IOptions <AntiPlagiarismConfiguration> configuration)
: base(clientsRepo, db)
{
this.submissionsRepo = submissionsRepo;
this.snippetsRepo = snippetsRepo;
this.tasksRepo = tasksRepo;
this.workQueueRepo = workQueueRepo;
this.mostSimilarSubmissionsRepo = mostSimilarSubmissionsRepo;
this.manualSuspicionLevelsRepo = manualSuspicionLevelsRepo;
this.plagiarismDetector = plagiarismDetector;
this.codeUnitsExtractor = codeUnitsExtractor;
this.newSubmissionHandler = newSubmissionHandler;
this.serviceScopeFactory = serviceScopeFactory;
this.configuration = configuration.Value;
}
private static async Task Run(AntiPlagiarismDb adb, UlearnDb db, IServiceProvider serviceProvider)
{
//await new UsersRepo(db, serviceProvider.GetService<UlearnUserManager>()).CreateUlearnBotUserIfNotExistsAsync();
//FillLanguageToAntiplagiarism.FillLanguage(adb);
//GenerateUpdateSequences();
//CompareColumns();
//await ResendLti(db);
//await FindExternalSolutionsPlagiarism.UploadSolutions();
//await FindExternalSolutionsPlagiarism.GetRawResults();
//await FindExternalSolutionsPlagiarism.PrepareResults();
//await UpdateExerciseVisits(db, "fpIntroduction");
//Users.PrintCourseAdmins(db);
//await ScoresUpdater.UpdateTests(db, "java-rtf");
//GetMostSimilarSubmission(adb);
//ParsePairWeightsFromLogs();
//GetBlackAndWhiteLabels(db, adb);
//ParseTaskWeightsFromLogs();
//CampusRegistration();
//GetIps(db);
//FillAntiplagFields.FillClientSubmissionId(adb);
//await XQueueRunAutomaticChecking(db);
//TextBlobsWithZeroByte(db);
//UpdateCertificateArchives(db);
//GetVKByEmail(serviceProvider);
}
private static void GetBlackAndWhiteLabels(UlearnDb db, AntiPlagiarismDb adb)
{
var lines = File.ReadLines("pairweights.txt");
var jsons = PlagiarismInstructorDecisions.GetBlackAndWhiteLabels(db, adb,
lines.Select(JsonConvert.DeserializeObject <BestPairWeight>));
File.WriteAllLines("blackAndWhiteLabels.txt", jsons);
}
private static IServiceProvider ConfigureDI(AntiPlagiarismDb adb, UlearnDb db)
{
var services = new ServiceCollection();
services.AddLogging(builder => builder.AddVostok(LogProvider.Get()));
services.AddSingleton(db);
services.AddDatabaseServices();
services.AddSingleton(adb);
services.AddIdentity <ApplicationUser, IdentityRole>().AddEntityFrameworkStores <UlearnDb>();
return(services.BuildServiceProvider());
}
private static Language?GetLanguageByTaskId(Guid taskId, AntiPlagiarismDb adb)
{
if (!taskIdToSubmission.ContainsKey(taskId))
{
var submission = adb.Submissions
.OrderByDescending(s => s.AddingTime)
.FirstOrDefault(s => s.TaskId == taskId);
taskIdToSubmission[taskId] = submission?.Language;
}
return(taskIdToSubmission[taskId]);
}
private static async Task UpdateOldSnippetsStatisticsForTaskBatch(DateTime @from, DateTime to, Guid taskId, int clientId,
AntiPlagiarismDb db, IQueryable <int> snippetIdsInNonInfluencingSubmissions,
List <SnippetStatistics> snippetsStatistics, string snippetsCountLog)
{
var snippet2AuthorsCount = (await db.SnippetsOccurences
.Where(s =>
s.Submission.AddingTime > to &&
s.Submission.TaskId == taskId && s.Submission.ClientId == clientId &&
snippetIdsInNonInfluencingSubmissions.Contains(s.SnippetId))
.Select(s => new { s.SnippetId, s.Submission.AuthorId })
.Distinct()
.GroupBy(o => new { o.SnippetId })
.Select(g => new { g.Key.SnippetId, AuthorsCount = g.Count() })
.ToListAsync())
.ToDictionary(p => p.SnippetId, p => p.AuthorsCount);
log.Info($"Authors counted for {snippetsCountLog} snippets for task {taskId} from {@from.ToSortable()} to {to.ToSortable()}");
await WriteNewAuthorsCounts(@from, to, taskId, db, snippetsStatistics, snippet2AuthorsCount);
}
public static IEnumerable <string> GetBlackAndWhiteLabels(UlearnDb db, AntiPlagiarismDb adb, IEnumerable <BestPairWeight> bestPairWeights)
{
var weightsDict = bestPairWeights.ToDictionary(e => e.Submission, e => e.Weight);
var blackLabels = GetBlackLabels(db).ToList();
Console.WriteLine($"Black labels count: {blackLabels.Count}");
var whiteLabels = GetWhiteLabels(db).ToList();
var concat = blackLabels.Concat(whiteLabels).ToList();
Console.WriteLine($"Labels count: {concat.Count}");
Thread.Sleep(5000);
var i = 0;
foreach (var e in concat)
{
var info1 = $"{{\"SubmissionId\": {e.SubmissionId}}}";
var info2 = $"{{\r\n \"SubmissionId\": {e.SubmissionId}\r\n}}";
var asi = adb.Submissions
.Where(s => s.AdditionalInfo == info1 || s.AdditionalInfo == info2)
.Select(s => new { s.Id })
.FirstOrDefault();
i++;
if (i % 1000 == 0)
{
Console.WriteLine($"i: {i}");
}
if (asi == null)
{
Console.WriteLine("asi is null");
continue;
}
e.AntiplagSubmissionId = asi.Id;
if (weightsDict.ContainsKey(e.AntiplagSubmissionId))
{
e.Weight = weightsDict[e.AntiplagSubmissionId];
}
}
return(concat
.Where(e => e.Weight != null)
.Select(JsonConvert.SerializeObject));
}
public ApiController(
AntiPlagiarismDb db,
ISubmissionsRepo submissionsRepo, ISnippetsRepo snippetsRepo, ITasksRepo tasksRepo, IClientsRepo clientsRepo,
StatisticsParametersFinder statisticsParametersFinder,
PlagiarismDetector plagiarismDetector,
CodeUnitsExtractor codeUnitsExtractor,
SubmissionSnippetsExtractor submissionSnippetsExtractor,
ILogger logger,
IServiceScopeFactory serviceScopeFactory,
IOptions <AntiPlagiarismConfiguration> configuration)
: base(logger, clientsRepo, db)
{
this.submissionsRepo = submissionsRepo;
this.snippetsRepo = snippetsRepo;
this.tasksRepo = tasksRepo;
this.statisticsParametersFinder = statisticsParametersFinder;
this.plagiarismDetector = plagiarismDetector;
this.codeUnitsExtractor = codeUnitsExtractor;
this.submissionSnippetsExtractor = submissionSnippetsExtractor;
this.serviceScopeFactory = serviceScopeFactory;
this.configuration = configuration.Value;
}
public static async Task Main(string[] args)
{
var configuration = ApplicationConfiguration.Read <UlearnConfiguration>();
LoggerSetup.Setup(configuration.HostLog, configuration.GraphiteServiceName);
try
{
var optionsBuilder = new DbContextOptionsBuilder <UlearnDb>()
.UseLazyLoadingProxies()
.UseNpgsql(configuration.Database, o => o.SetPostgresVersion(13, 2));
var db = new UlearnDb(optionsBuilder.Options);
var aOptionsBuilder = new DbContextOptionsBuilder <AntiPlagiarismDb>()
.UseLazyLoadingProxies()
.UseNpgsql(configuration.Database, o => o.SetPostgresVersion(13, 2));
var adb = new AntiPlagiarismDb(aOptionsBuilder.Options);
var serviceProvider = ConfigureDI(adb, db);
await Run(adb, db, serviceProvider);
}
finally
{
await FileLog.FlushAllAsync();
}
}
public static void FillLanguage(AntiPlagiarismDb adb)
{
FillLanguageManualSuspicionLevels(adb);
FillLanguageTasksStatisticsParameters(adb);
FillLanguageSnippetsStatistics(adb, true);
}
private static void FillLanguageSnippetsStatistics(AntiPlagiarismDb adb, bool @do)
{
Console.WriteLine("FillLanguageSnippetsStatistics");
var taskIds = adb
.SnippetsStatistics
.Select(s => s.TaskId)
.Distinct()
.ToList();
Console.WriteLine($"Count taskIds {taskIds.Count}");
var taskIdToLanguage = new Dictionary <Guid, Language?>();
foreach (var taskId in taskIds)
{
if (!taskIdToLanguage.ContainsKey(taskId))
{
taskIdToLanguage[taskId] = GetLanguageByTaskId(taskId, adb);
}
}
var snippets = adb
.SnippetsStatistics
.Select(c => new { c.Id, c.TaskId, c.Language })
.AsNoTracking();
var count = snippets.Count();
Console.WriteLine($"Count snippets {count}");
var changesCount = 0;
var changes = new List <(int Id, Language Language)>();
using (var changesFile = new StreamWriter("changes.txt"))
{
var getChangesCompleted = 0;
foreach (var snippet in snippets)
{
getChangesCompleted++;
if (getChangesCompleted % 10000 == 0)
{
Console.WriteLine($"getChangesCompleted {getChangesCompleted} / {count}");
}
var newLanguage = taskIdToLanguage[snippet.TaskId];
if (newLanguage == null)
{
newLanguage = 0;
}
if (newLanguage == snippet.Language)
{
continue;
}
changesFile.WriteLine($"{snippet.Id} {newLanguage.Value}");
changes.Add((snippet.Id, newLanguage.Value));
changesCount++;
}
}
Console.WriteLine($"Found changes {changesCount}");
var batchSize = 300;
var currentBatch = 0;
var batches = new List <List <(int Id, Language Language)> >();
while (currentBatch * batchSize < changes.Count)
{
var nextBatch = changes.Skip(currentBatch * batchSize)
.Take(batchSize).ToList();
batches.Add(nextBatch);
currentBatch++;
}
if (@do)
{
adb.DisableAutoDetectChanges();
var completed = 0;
foreach (var batch in batches)
{
var batchIds = batch.Select(s => s.Id).ToList();
var batchDict = batch.ToDictionary(s => s.Id, s => s.Language);
var batchData = adb.SnippetsStatistics.Where(s => batchIds.Contains(s.Id)).ToList();
foreach (var data in batchData)
{
data.Language = batchDict[data.Id];
adb.SnippetsStatistics.Update(data);
}
completed += batch.Count;
Console.WriteLine($"FillLanguageSnippetsStatistics - Completed {completed} / {changesCount}");
adb.SaveChanges();
}
adb.EnableAutoDetectChanges();
}
}
public SubmissionsRepo(AntiPlagiarismDb db)
{
this.db = db;
}
public TasksRepo(AntiPlagiarismDb db)
{
this.db = db;
}
public ClientsRepo(AntiPlagiarismDb db)
{
this.db = db;
}
protected BaseController(IClientsRepo clientsRepo, AntiPlagiarismDb db)
{
this.clientsRepo = clientsRepo;
this.db = db;
}
public SnippetsRepo(AntiPlagiarismDb db, IServiceScopeFactory serviceScopeFactory)
{
this.db = db;
this.serviceScopeFactory = serviceScopeFactory;
}
public WorkQueueRepo(AntiPlagiarismDb db)
{
this.db = db;
}
protected BaseController(ILogger logger, IClientsRepo clientsRepo, AntiPlagiarismDb db)
{
this.logger = logger;
this.clientsRepo = clientsRepo;
this.db = db;
}
public SnippetsRepo(AntiPlagiarismDb db, ILogger logger)
{
this.db = db;
this.logger = logger;
}
private static async Task WriteNewAuthorsCounts(DateTime @from, DateTime to, Guid taskId, AntiPlagiarismDb db,
List <SnippetStatistics> snippetsStatistics, Dictionary <int, int> snippet2AuthorsCount)
{
foreach (var snippetStatistics in snippetsStatistics)
{
// snippet2AuthorsCount не содержит ключа, если с момента to не было посылок c этим сниппетом
snippetStatistics.AuthorsCount = snippet2AuthorsCount.GetValueOrDefault(snippetStatistics.SnippetId);
db.Entry(snippetStatistics).State = EntityState.Modified;
}
await db.SaveChangesAsync();
foreach (var snippetStatistics in snippetsStatistics)
{
db.Entry(snippetStatistics).State = EntityState.Unchanged;
}
log.Info($"New authors count are written for {snippetsStatistics.Count} snippets for task {taskId} from {@from.ToSortable()} to {to.ToSortable()}");
}
public ManualSuspicionLevelsRepo(AntiPlagiarismDb db)
{
this.db = db;
}
public MostSimilarSubmissionsRepo(AntiPlagiarismDb db)
{
this.db = db;
}
