public void MatchSucceedsWithDenormalizedCorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "1ab C 1"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "2xyZ 2"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "1 A B C 1", "2 X Y Z 2" }
};
var match = new Match(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.True(grade.Success);
}
}
public void MatchFailsWithIncorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "1111"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "2222"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "aaaa", "bbbb" }
};
var match = new Match(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
}
}
public void MatchAnySucceedsWithCorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "ONE|TWO|THREE"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "FOUR|FIVE|SIX"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "three", "five" }
};
var match = new MatchAny(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.True(grade.Success);
}
}
public void MatchAnyFailsWithPartialCredit()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 10, Value = "OnE|TWO|ThReE"
},
new TokenSpec {
Label = "2", Percent = 90, Value = "4|5|6"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "two", "J" }
};
var match = new MatchAny(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
Assert.Equal(10, grade.CorrectPercent);
}
}
public void MatchAlphaNumericFailsWithIncorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "1111"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "2222"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "@%@#%@#%@#%@^aaaa^^#@$^@$!#$!", "#%@^@#$^@#$@#$bbbb@%^@#^@%^@#$%@#" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
}
}
public void MatchAlphaNumericFailsWithPartialCredit()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 10, Value = "1111Aa"
},
new TokenSpec {
Label = "2", Percent = 90, Value = "2222Bb"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "^%@$#^$%@#^$%@^$@^#$@%#$%#$@%#@$%", "2222!Bb" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
Assert.Equal(90, grade.CorrectPercent);
}
}
public void MatchAlphaNumericSucceedsWithCorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "1111Aa"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "2222Bb"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "!$!@$!$%!%1111^%$^$%^$%^$%^Aa", "#@%$@#%^@#^$#$^2222@^#$^@$!@$!$Bb" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.True(grade.Success);
}
}
public void MatchAllFailsWithIncorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "THREE|TWO|ONE"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "SIX|FIVE|FOUR"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "one, two, four", "three five six" }
};
var match = new MatchAll(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
}
}
public void MatchAllFailsWithPartialCredit()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 60, Value = "THREE|TWO|ONE"
},
new TokenSpec {
Label = "2", Percent = 40, Value = "SIX|FIVE|FOUR"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "one is smaller than two is smaller than three", "four is less than five is less than seven" }
};
var match = new MatchAll(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
Assert.Equal(60, grade.CorrectPercent);
}
}
/// <summary>
/// Save submitted flag
/// </summary>
/// <param name="context"></param>
/// <param name="flag"></param>
private void SaveFlag(ProblemContext context, ProblemFlag flag)
{
string fn = Path.Combine(context.ProblemFolder, flag.SubmissionId + ".json");
string data = JsonConvert.SerializeObject(flag);
File.WriteAllText(fn, data);
}
public async Task Grade(ProblemFlag flag)
{
var result = await Http.PutAsync("/api/problem", Json(flag));
if (!result.IsSuccessStatusCode)
{
throw new HttpRequestException();
}
}
public Task Grade(ProblemFlag flag)
{
bool passed = false;
if (!Cache.TryGetValue(Key(flag.Id), out ProblemState state))
{
throw new Exception("Problem not found.");
}
if (state.End != null && state.End != DateTime.MinValue)
{
throw new Exception("Problem already complete.");
}
Logger.LogDebug("received flag");
var graded = new GradedSubmission
{
SubmissionId = flag.SubmissionId,
ProblemId = flag.Id,
Timestamp = DateTime.UtcNow,
Status = SubmissionStatus.Submitted,
State = state
};
Task.WaitAll(Handler.Update(graded));
Logger.LogDebug("grading flag");
if (Cache.TryGetValue(FlagKey(flag.Id), out string target))
{
passed = flag.Tokens[0] == target;
}
if (passed || flag.Count >= 3)
{
state.End = DateTime.UtcNow;
state.Status = passed ? ProblemStatus.Success : ProblemStatus.Failure;
state.Percent = passed ? 10 : 0;
state.GamespaceReady = false;
}
graded.Status = passed ? SubmissionStatus.Passed : SubmissionStatus.Failed;
Task.Delay(new Random().Next(2000, 6000)).Wait();
Logger.LogDebug("graded flag");
Task.WaitAll(Handler.Update(graded));
return(Task.CompletedTask);
}
public IActionResult Put([FromBody] ProblemFlag model)
{
var qm = new ClientProblemFlag
{
Id = model.Id,
Client = HttpContext.Request.Headers["X-API-CLIENT"],
CallbackUrl = HttpContext.Request.Headers["X-API-CALLBACK"],
ProblemFlag = model
};
_flagQueue.Enqueue(qm);
return(Ok());
}
public void MatchAlphaNumericSucceedsWithCorrectSingleValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 100, Value = "123.456-7&8"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "..123~456~78" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.True(grade.Success);
}
}
public GradingResult GradeTokens(ProblemFlag problemFlag)
{
if (ProblemContext.Flag.Tokens.Length == 0)
{
throw new ProblemGradingException("No tokens defined. Problem Id: " + ProblemContext.Problem.Id);
}
if (problemFlag.Tokens.Length != ProblemContext.Flag.Tokens.Length)
{
throw new ProblemGradingException("The number of tokens submitted is not the same as the number of tokens for the challenge. Submitted token: " +
string.Join(" ", problemFlag.Tokens) + " - Problem Id: " + ProblemContext.Problem.Id);
}
var result = new GradingResult(ProblemContext);
var gradedTokens = new List <Token>();
var timestamp = DateTime.UtcNow;
var tokens = Unwrap
? problemFlag.Tokens.Select(t => t.Unwrap(Options))
: problemFlag.Tokens;
int index = 0;
foreach (string token in tokens)
{
var existing = GetExistingToken(token, index);
if (existing == null)
{
var spec = ProblemContext.Flag.Tokens[index];
if (spec == null)
{
throw new TokenSpecNotFoundException();
}
var match = GradeToken(spec, token);
var percent = match
? (ProblemContext.Flag.Tokens.Length == 1 ? 100 : spec.Percent)
: 0;
gradedTokens.Add(
new Token
{
Value = token,
Percent = percent,
Status = match ? TokenStatusType.Correct : TokenStatusType.Incorrect,
Timestamp = timestamp,
Index = index
});
if (ProblemContext.Spec.IsMultiStage)
{
break;
}
}
else
{
gradedTokens.Add(existing);
}
index++;
}
result.GradedTokens = gradedTokens.OrderBy(t => t.Index).ToList();
return(result);
}
/// <summary>
/// Grades a flag submitted for a problem
/// </summary>
/// <param name="flag"></param>
/// <returns></returns>
public GradedSubmission Grade(ProblemFlag flag)
{
ProblemContext context = null;
GradingResult result = null;
Logger.LogDebug($"grading {flag.Id}");
var submissionStatus = SubmissionStatus.Submitted;
try
{
context = Data.LoadContext(flag.Id);
if (context == null || context.Flag == null)
{
throw new NotFoundException();
}
if (context.ProblemState.Status == ProblemStatus.Complete)
{
throw new ProblemCompleteException();
}
SaveFlag(context, flag);
IGradingStrategy strategy = null;
switch (context.Flag.Type)
{
case FlagType.MatchOutput:
strategy = new MatchOutput(Options, Logger, context);
break;
case FlagType.MatchAll:
strategy = new MatchAll(Options, Logger, context);
break;
case FlagType.MatchAny:
strategy = new MatchAny(Options, Logger, context);
break;
case FlagType.Match:
strategy = new Match(Options, Logger, context);
break;
case FlagType.MatchAlphaNumeric:
default:
strategy = new MatchAlphaNumeric(Options, Logger, context);
break;
}
result = strategy.GradeTokens(flag);
}
catch (Exception ex)
{
Logger.LogError(ex, "Had a problem with grading");
}
finally
{
if (context != null && result != null)
{
submissionStatus = result.Success ? SubmissionStatus.Passed : SubmissionStatus.Failed;
var check = false;
// if max submissions <= 0 then we accept
// unlimited submissions if unsuccessful
if (context.Problem.Settings.MaxSubmissions > 0)
{
if (context.Spec.IsMultiStage)
{
// if multi stage we only consider the
// last graded token incorrect count
check = result.GradedTokens.Last().Status != TokenStatusType.Correct &&
flag.Count >= context.Problem.Settings.MaxSubmissions;
}
else
{
check = flag.Count >= context.Problem.Settings.MaxSubmissions;
}
}
var isFinal = result.Success || check;
if (isFinal)
{
//_ = Mojo.Stop(flag.Id);
context.ProblemState.End = DateTime.UtcNow;
//context.ProblemState.GamespaceReady = false;
context.ProblemState.Status = result.Success ? ProblemStatus.Success : ProblemStatus.Failure;
}
context.ProblemState.Percent = result.CorrectPercent;
// map TokenSpec detail along with correct answers
var problemStateTokens = new List <Token>();
int index = 0;
foreach (var tokenSpec in context.Flag.Tokens)
{
var correct = result.GradedTokens.SingleOrDefault(t => t.Index == index && t.Status == TokenStatusType.Correct);
problemStateTokens.Add(new Token
{
Index = index,
Label = tokenSpec.Label,
Percent = context.Flag.Tokens.Length == 1 ? 100 : tokenSpec.Percent,
Status = correct?.Status ?? TokenStatusType.Pending,
Timestamp = correct?.Timestamp,
Value = correct?.Value
});
index++;
}
context.ProblemState.Tokens = problemStateTokens;
Data.SaveContext(context);
}
}
Logger.LogDebug($"returning {flag.Id}");
return(new GradedSubmission
{
ProblemId = flag.Id,
SubmissionId = flag.SubmissionId,
Status = submissionStatus,
State = context?.ProblemState,
Timestamp = DateTime.UtcNow,
Tokens = result?.GradedTokens
});
}
