public void MatchAlphaNumericFailsWithPartialCredit()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 10, Value = "1111Aa"
},
new TokenSpec {
Label = "2", Percent = 90, Value = "2222Bb"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "^%@$#^$%@#^$%@^$@^#$@%#$%#$@%#@$%", "2222!Bb" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
Assert.Equal(90, grade.CorrectPercent);
}
}
public void MatchAlphaNumericFailsWithIncorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "1111"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "2222"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "@%@#%@#%@#%@^aaaa^^#@$^@$!#$!", "#%@^@#$^@#$@#$bbbb@%^@#^@%^@#$%@#" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.False(grade.Success);
}
}
public void MatchAlphaNumericSucceedsWithCorrectValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 50, Value = "1111Aa"
},
new TokenSpec {
Label = "2", Percent = 50, Value = "2222Bb"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "!$!@$!$%!%1111^%$^$%^$%^$%^Aa", "#@%$@#%^@#^$#$^2222@^#$^@$!@$!$Bb" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.True(grade.Success);
}
}
public void MatchAlphaNumericSucceedsWithCorrectSingleValues()
{
using (var context = new GradingContext())
{
var problemContext = context.MockProblemContext(100,
new TokenSpec {
Label = "1", Percent = 100, Value = "123.456-7&8"
}
);
var problemFlag = new ProblemFlag
{
Id = Guid.NewGuid().ToString(),
SubmissionId = Guid.NewGuid().ToString(),
Tokens = new string[] { "..123~456~78" }
};
var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);
var grade = match.GradeTokens(problemFlag);
Assert.True(grade.Success);
}
}
/// <summary>
/// Grades a flag submitted for a problem
/// </summary>
/// <param name="flag"></param>
/// <returns></returns>
public GradedSubmission Grade(ProblemFlag flag)
{
ProblemContext context = null;
GradingResult result = null;
Logger.LogDebug($"grading {flag.Id}");
var submissionStatus = SubmissionStatus.Submitted;
try
{
context = Data.LoadContext(flag.Id);
if (context == null || context.Flag == null)
{
throw new NotFoundException();
}
if (context.ProblemState.Status == ProblemStatus.Complete)
{
throw new ProblemCompleteException();
}
SaveFlag(context, flag);
IGradingStrategy strategy = null;
switch (context.Flag.Type)
{
case FlagType.MatchOutput:
strategy = new MatchOutput(Options, Logger, context);
break;
case FlagType.MatchAll:
strategy = new MatchAll(Options, Logger, context);
break;
case FlagType.MatchAny:
strategy = new MatchAny(Options, Logger, context);
break;
case FlagType.Match:
strategy = new Match(Options, Logger, context);
break;
case FlagType.MatchAlphaNumeric:
default:
strategy = new MatchAlphaNumeric(Options, Logger, context);
break;
}
result = strategy.GradeTokens(flag);
}
catch (Exception ex)
{
Logger.LogError(ex, "Had a problem with grading");
}
finally
{
if (context != null && result != null)
{
submissionStatus = result.Success ? SubmissionStatus.Passed : SubmissionStatus.Failed;
var check = false;
// if max submissions <= 0 then we accept
// unlimited submissions if unsuccessful
if (context.Problem.Settings.MaxSubmissions > 0)
{
if (context.Spec.IsMultiStage)
{
// if multi stage we only consider the
// last graded token incorrect count
check = result.GradedTokens.Last().Status != TokenStatusType.Correct &&
flag.Count >= context.Problem.Settings.MaxSubmissions;
}
else
{
check = flag.Count >= context.Problem.Settings.MaxSubmissions;
}
}
var isFinal = result.Success || check;
if (isFinal)
{
//_ = Mojo.Stop(flag.Id);
context.ProblemState.End = DateTime.UtcNow;
//context.ProblemState.GamespaceReady = false;
context.ProblemState.Status = result.Success ? ProblemStatus.Success : ProblemStatus.Failure;
}
context.ProblemState.Percent = result.CorrectPercent;
// map TokenSpec detail along with correct answers
var problemStateTokens = new List <Token>();
int index = 0;
foreach (var tokenSpec in context.Flag.Tokens)
{
var correct = result.GradedTokens.SingleOrDefault(t => t.Index == index && t.Status == TokenStatusType.Correct);
problemStateTokens.Add(new Token
{
Index = index,
Label = tokenSpec.Label,
Percent = context.Flag.Tokens.Length == 1 ? 100 : tokenSpec.Percent,
Status = correct?.Status ?? TokenStatusType.Pending,
Timestamp = correct?.Timestamp,
Value = correct?.Value
});
index++;
}
context.ProblemState.Tokens = problemStateTokens;
Data.SaveContext(context);
}
}
Logger.LogDebug($"returning {flag.Id}");
return(new GradedSubmission
{
ProblemId = flag.Id,
SubmissionId = flag.SubmissionId,
Status = submissionStatus,
State = context?.ProblemState,
Timestamp = DateTime.UtcNow,
Tokens = result?.GradedTokens
});
}
