public void MatchSucceedsWithDenormalizedCorrectValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 50, Value = "1ab C 1" }, new TokenSpec { Label = "2", Percent = 50, Value = "2xyZ 2" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "1 A B C 1", "2 X Y Z 2" } }; var match = new Match(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.True(grade.Success); } }
public void MatchFailsWithIncorrectValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 50, Value = "1111" }, new TokenSpec { Label = "2", Percent = 50, Value = "2222" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "aaaa", "bbbb" } }; var match = new Match(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.False(grade.Success); } }
public void MatchAnySucceedsWithCorrectValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 50, Value = "ONE|TWO|THREE" }, new TokenSpec { Label = "2", Percent = 50, Value = "FOUR|FIVE|SIX" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "three", "five" } }; var match = new MatchAny(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.True(grade.Success); } }
public void MatchAnyFailsWithPartialCredit() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 10, Value = "OnE|TWO|ThReE" }, new TokenSpec { Label = "2", Percent = 90, Value = "4|5|6" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "two", "J" } }; var match = new MatchAny(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.False(grade.Success); Assert.Equal(10, grade.CorrectPercent); } }
public void MatchAlphaNumericFailsWithIncorrectValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 50, Value = "1111" }, new TokenSpec { Label = "2", Percent = 50, Value = "2222" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "@%@#%@#%@#%@^aaaa^^#@$^@$!#$!", "#%@^@#$^@#$@#$bbbb@%^@#^@%^@#$%@#" } }; var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.False(grade.Success); } }
public void MatchAlphaNumericFailsWithPartialCredit() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 10, Value = "1111Aa" }, new TokenSpec { Label = "2", Percent = 90, Value = "2222Bb" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "^%@$#^$%@#^$%@^$@^#$@%#$%#$@%#@$%", "2222!Bb" } }; var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.False(grade.Success); Assert.Equal(90, grade.CorrectPercent); } }
public void MatchAlphaNumericSucceedsWithCorrectValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 50, Value = "1111Aa" }, new TokenSpec { Label = "2", Percent = 50, Value = "2222Bb" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "!$!@$!$%!%1111^%$^$%^$%^$%^Aa", "#@%$@#%^@#^$#$^2222@^#$^@$!@$!$Bb" } }; var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.True(grade.Success); } }
public void MatchAllFailsWithIncorrectValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 50, Value = "THREE|TWO|ONE" }, new TokenSpec { Label = "2", Percent = 50, Value = "SIX|FIVE|FOUR" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "one, two, four", "three five six" } }; var match = new MatchAll(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.False(grade.Success); } }
public void MatchAllFailsWithPartialCredit() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 60, Value = "THREE|TWO|ONE" }, new TokenSpec { Label = "2", Percent = 40, Value = "SIX|FIVE|FOUR" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "one is smaller than two is smaller than three", "four is less than five is less than seven" } }; var match = new MatchAll(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.False(grade.Success); Assert.Equal(60, grade.CorrectPercent); } }
/// <summary> /// Save submitted flag /// </summary> /// <param name="context"></param> /// <param name="flag"></param> private void SaveFlag(ProblemContext context, ProblemFlag flag) { string fn = Path.Combine(context.ProblemFolder, flag.SubmissionId + ".json"); string data = JsonConvert.SerializeObject(flag); File.WriteAllText(fn, data); }
public async Task Grade(ProblemFlag flag) { var result = await Http.PutAsync("/api/problem", Json(flag)); if (!result.IsSuccessStatusCode) { throw new HttpRequestException(); } }
public Task Grade(ProblemFlag flag) { bool passed = false; if (!Cache.TryGetValue(Key(flag.Id), out ProblemState state)) { throw new Exception("Problem not found."); } if (state.End != null && state.End != DateTime.MinValue) { throw new Exception("Problem already complete."); } Logger.LogDebug("received flag"); var graded = new GradedSubmission { SubmissionId = flag.SubmissionId, ProblemId = flag.Id, Timestamp = DateTime.UtcNow, Status = SubmissionStatus.Submitted, State = state }; Task.WaitAll(Handler.Update(graded)); Logger.LogDebug("grading flag"); if (Cache.TryGetValue(FlagKey(flag.Id), out string target)) { passed = flag.Tokens[0] == target; } if (passed || flag.Count >= 3) { state.End = DateTime.UtcNow; state.Status = passed ? ProblemStatus.Success : ProblemStatus.Failure; state.Percent = passed ? 10 : 0; state.GamespaceReady = false; } graded.Status = passed ? SubmissionStatus.Passed : SubmissionStatus.Failed; Task.Delay(new Random().Next(2000, 6000)).Wait(); Logger.LogDebug("graded flag"); Task.WaitAll(Handler.Update(graded)); return(Task.CompletedTask); }
public IActionResult Put([FromBody] ProblemFlag model) { var qm = new ClientProblemFlag { Id = model.Id, Client = HttpContext.Request.Headers["X-API-CLIENT"], CallbackUrl = HttpContext.Request.Headers["X-API-CALLBACK"], ProblemFlag = model }; _flagQueue.Enqueue(qm); return(Ok()); }
public void MatchAlphaNumericSucceedsWithCorrectSingleValues() { using (var context = new GradingContext()) { var problemContext = context.MockProblemContext(100, new TokenSpec { Label = "1", Percent = 100, Value = "123.456-7&8" } ); var problemFlag = new ProblemFlag { Id = Guid.NewGuid().ToString(), SubmissionId = Guid.NewGuid().ToString(), Tokens = new string[] { "..123~456~78" } }; var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext); var grade = match.GradeTokens(problemFlag); Assert.True(grade.Success); } }
public GradingResult GradeTokens(ProblemFlag problemFlag) { if (ProblemContext.Flag.Tokens.Length == 0) { throw new ProblemGradingException("No tokens defined. Problem Id: " + ProblemContext.Problem.Id); } if (problemFlag.Tokens.Length != ProblemContext.Flag.Tokens.Length) { throw new ProblemGradingException("The number of tokens submitted is not the same as the number of tokens for the challenge. Submitted token: " + string.Join(" ", problemFlag.Tokens) + " - Problem Id: " + ProblemContext.Problem.Id); } var result = new GradingResult(ProblemContext); var gradedTokens = new List <Token>(); var timestamp = DateTime.UtcNow; var tokens = Unwrap ? problemFlag.Tokens.Select(t => t.Unwrap(Options)) : problemFlag.Tokens; int index = 0; foreach (string token in tokens) { var existing = GetExistingToken(token, index); if (existing == null) { var spec = ProblemContext.Flag.Tokens[index]; if (spec == null) { throw new TokenSpecNotFoundException(); } var match = GradeToken(spec, token); var percent = match ? (ProblemContext.Flag.Tokens.Length == 1 ? 100 : spec.Percent) : 0; gradedTokens.Add( new Token { Value = token, Percent = percent, Status = match ? TokenStatusType.Correct : TokenStatusType.Incorrect, Timestamp = timestamp, Index = index }); if (ProblemContext.Spec.IsMultiStage) { break; } } else { gradedTokens.Add(existing); } index++; } result.GradedTokens = gradedTokens.OrderBy(t => t.Index).ToList(); return(result); }
/// <summary> /// Grades a flag submitted for a problem /// </summary> /// <param name="flag"></param> /// <returns></returns> public GradedSubmission Grade(ProblemFlag flag) { ProblemContext context = null; GradingResult result = null; Logger.LogDebug($"grading {flag.Id}"); var submissionStatus = SubmissionStatus.Submitted; try { context = Data.LoadContext(flag.Id); if (context == null || context.Flag == null) { throw new NotFoundException(); } if (context.ProblemState.Status == ProblemStatus.Complete) { throw new ProblemCompleteException(); } SaveFlag(context, flag); IGradingStrategy strategy = null; switch (context.Flag.Type) { case FlagType.MatchOutput: strategy = new MatchOutput(Options, Logger, context); break; case FlagType.MatchAll: strategy = new MatchAll(Options, Logger, context); break; case FlagType.MatchAny: strategy = new MatchAny(Options, Logger, context); break; case FlagType.Match: strategy = new Match(Options, Logger, context); break; case FlagType.MatchAlphaNumeric: default: strategy = new MatchAlphaNumeric(Options, Logger, context); break; } result = strategy.GradeTokens(flag); } catch (Exception ex) { Logger.LogError(ex, "Had a problem with grading"); } finally { if (context != null && result != null) { submissionStatus = result.Success ? SubmissionStatus.Passed : SubmissionStatus.Failed; var check = false; // if max submissions <= 0 then we accept // unlimited submissions if unsuccessful if (context.Problem.Settings.MaxSubmissions > 0) { if (context.Spec.IsMultiStage) { // if multi stage we only consider the // last graded token incorrect count check = result.GradedTokens.Last().Status != TokenStatusType.Correct && flag.Count >= context.Problem.Settings.MaxSubmissions; } else { check = flag.Count >= context.Problem.Settings.MaxSubmissions; } } var isFinal = result.Success || check; if (isFinal) { //_ = Mojo.Stop(flag.Id); context.ProblemState.End = DateTime.UtcNow; //context.ProblemState.GamespaceReady = false; context.ProblemState.Status = result.Success ? ProblemStatus.Success : ProblemStatus.Failure; } context.ProblemState.Percent = result.CorrectPercent; // map TokenSpec detail along with correct answers var problemStateTokens = new List <Token>(); int index = 0; foreach (var tokenSpec in context.Flag.Tokens) { var correct = result.GradedTokens.SingleOrDefault(t => t.Index == index && t.Status == TokenStatusType.Correct); problemStateTokens.Add(new Token { Index = index, Label = tokenSpec.Label, Percent = context.Flag.Tokens.Length == 1 ? 100 : tokenSpec.Percent, Status = correct?.Status ?? TokenStatusType.Pending, Timestamp = correct?.Timestamp, Value = correct?.Value }); index++; } context.ProblemState.Tokens = problemStateTokens; Data.SaveContext(context); } } Logger.LogDebug($"returning {flag.Id}"); return(new GradedSubmission { ProblemId = flag.Id, SubmissionId = flag.SubmissionId, Status = submissionStatus, State = context?.ProblemState, Timestamp = DateTime.UtcNow, Tokens = result?.GradedTokens }); }