public void MatchAlphaNumericFailsWithPartialCredit()
        {
            using (var context = new GradingContext())
            {
                var problemContext = context.MockProblemContext(100,
                                                                new TokenSpec {
                    Label = "1", Percent = 10, Value = "1111Aa"
                },
                                                                new TokenSpec {
                    Label = "2", Percent = 90, Value = "2222Bb"
                }
                                                                );

                var problemFlag = new ProblemFlag
                {
                    Id           = Guid.NewGuid().ToString(),
                    SubmissionId = Guid.NewGuid().ToString(),
                    Tokens       = new string[] { "^%@$#^$%@#^$%@^$@^#$@%#$%#$@%#@$%", "2222!Bb" }
                };

                var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);

                var grade = match.GradeTokens(problemFlag);

                Assert.False(grade.Success);

                Assert.Equal(90, grade.CorrectPercent);
            }
        }
        public void MatchAlphaNumericFailsWithIncorrectValues()
        {
            using (var context = new GradingContext())
            {
                var problemContext = context.MockProblemContext(100,
                                                                new TokenSpec {
                    Label = "1", Percent = 50, Value = "1111"
                },
                                                                new TokenSpec {
                    Label = "2", Percent = 50, Value = "2222"
                }
                                                                );

                var problemFlag = new ProblemFlag
                {
                    Id           = Guid.NewGuid().ToString(),
                    SubmissionId = Guid.NewGuid().ToString(),
                    Tokens       = new string[] { "@%@#%@#%@#%@^aaaa^^#@$^@$!#$!", "#%@^@#$^@#$@#$bbbb@%^@#^@%^@#$%@#" }
                };

                var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);

                var grade = match.GradeTokens(problemFlag);

                Assert.False(grade.Success);
            }
        }
        public void MatchAlphaNumericSucceedsWithCorrectValues()
        {
            using (var context = new GradingContext())
            {
                var problemContext = context.MockProblemContext(100,
                                                                new TokenSpec {
                    Label = "1", Percent = 50, Value = "1111Aa"
                },
                                                                new TokenSpec {
                    Label = "2", Percent = 50, Value = "2222Bb"
                }
                                                                );

                var problemFlag = new ProblemFlag
                {
                    Id           = Guid.NewGuid().ToString(),
                    SubmissionId = Guid.NewGuid().ToString(),
                    Tokens       = new string[] { "!$!@$!$%!%1111^%$^$%^$%^$%^Aa", "#@%$@#%^@#^$#$^2222@^#$^@$!@$!$Bb" }
                };

                var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);

                var grade = match.GradeTokens(problemFlag);

                Assert.True(grade.Success);
            }
        }
        public void MatchAlphaNumericSucceedsWithCorrectSingleValues()
        {
            using (var context = new GradingContext())
            {
                var problemContext = context.MockProblemContext(100,
                                                                new TokenSpec {
                    Label = "1", Percent = 100, Value = "123.456-7&8"
                }
                                                                );

                var problemFlag = new ProblemFlag
                {
                    Id           = Guid.NewGuid().ToString(),
                    SubmissionId = Guid.NewGuid().ToString(),
                    Tokens       = new string[] { "..123~456~78" }
                };

                var match = new MatchAlphaNumeric(context.Options, context.Logger, problemContext);

                var grade = match.GradeTokens(problemFlag);

                Assert.True(grade.Success);
            }
        }
Example #5
0
        /// <summary>
        /// Grades a flag submitted for a problem
        /// </summary>
        /// <param name="flag"></param>
        /// <returns></returns>
        public GradedSubmission Grade(ProblemFlag flag)
        {
            ProblemContext context = null;
            GradingResult  result  = null;

            Logger.LogDebug($"grading {flag.Id}");
            var submissionStatus = SubmissionStatus.Submitted;

            try
            {
                context = Data.LoadContext(flag.Id);

                if (context == null || context.Flag == null)
                {
                    throw new NotFoundException();
                }

                if (context.ProblemState.Status == ProblemStatus.Complete)
                {
                    throw new ProblemCompleteException();
                }

                SaveFlag(context, flag);

                IGradingStrategy strategy = null;

                switch (context.Flag.Type)
                {
                case FlagType.MatchOutput:
                    strategy = new MatchOutput(Options, Logger, context);
                    break;

                case FlagType.MatchAll:
                    strategy = new MatchAll(Options, Logger, context);
                    break;

                case FlagType.MatchAny:
                    strategy = new MatchAny(Options, Logger, context);
                    break;

                case FlagType.Match:
                    strategy = new Match(Options, Logger, context);
                    break;

                case FlagType.MatchAlphaNumeric:
                default:
                    strategy = new MatchAlphaNumeric(Options, Logger, context);
                    break;
                }

                result = strategy.GradeTokens(flag);
            }
            catch (Exception ex)
            {
                Logger.LogError(ex, "Had a problem with grading");
            }
            finally
            {
                if (context != null && result != null)
                {
                    submissionStatus = result.Success ? SubmissionStatus.Passed : SubmissionStatus.Failed;

                    var check = false;

                    // if max submissions <= 0 then we accept
                    // unlimited submissions if unsuccessful
                    if (context.Problem.Settings.MaxSubmissions > 0)
                    {
                        if (context.Spec.IsMultiStage)
                        {
                            // if multi stage we only consider the
                            // last graded token incorrect count
                            check = result.GradedTokens.Last().Status != TokenStatusType.Correct &&
                                    flag.Count >= context.Problem.Settings.MaxSubmissions;
                        }
                        else
                        {
                            check = flag.Count >= context.Problem.Settings.MaxSubmissions;
                        }
                    }

                    var isFinal = result.Success || check;

                    if (isFinal)
                    {
                        //_ = Mojo.Stop(flag.Id);

                        context.ProblemState.End = DateTime.UtcNow;
                        //context.ProblemState.GamespaceReady = false;
                        context.ProblemState.Status = result.Success ? ProblemStatus.Success : ProblemStatus.Failure;
                    }

                    context.ProblemState.Percent = result.CorrectPercent;

                    // map TokenSpec detail along with correct answers
                    var problemStateTokens = new List <Token>();

                    int index = 0;
                    foreach (var tokenSpec in context.Flag.Tokens)
                    {
                        var correct = result.GradedTokens.SingleOrDefault(t => t.Index == index && t.Status == TokenStatusType.Correct);

                        problemStateTokens.Add(new Token
                        {
                            Index     = index,
                            Label     = tokenSpec.Label,
                            Percent   = context.Flag.Tokens.Length == 1 ? 100 : tokenSpec.Percent,
                            Status    = correct?.Status ?? TokenStatusType.Pending,
                            Timestamp = correct?.Timestamp,
                            Value     = correct?.Value
                        });

                        index++;
                    }

                    context.ProblemState.Tokens = problemStateTokens;

                    Data.SaveContext(context);
                }
            }

            Logger.LogDebug($"returning {flag.Id}");

            return(new GradedSubmission
            {
                ProblemId = flag.Id,
                SubmissionId = flag.SubmissionId,
                Status = submissionStatus,
                State = context?.ProblemState,
                Timestamp = DateTime.UtcNow,
                Tokens = result?.GradedTokens
            });
        }