/// <summary>
        /// Computes the ratio of the symmetric difference to the size of dfa1 enumerating paths up to length n (uses the complement if density is high)
        /// </summary>
        /// <returns>size of ((dfa2-dfa1)+(dfa1-dfa2))/dfa1</returns>
        public static double GetDFADifferenceRatio(Automaton <BDD> dfa1, Automaton <BDD> dfa2, HashSet <char> al, CharSetSolver solver)
        {
            var solutionDensity = DFADensity.GetDFADensity(dfa1, al, solver);

            //Symmetric difference
            var dfadiff1 = dfa1.Minus(dfa2, solver);
            var dfadiff2 = dfa2.Minus(dfa1, solver);
            var dfatrue  = Automaton <BDD> .Create(0, new int[] { 0 }, new Move <BDD>[] { new Move <BDD>(0, 0, solver.True) });

            var dfadiff = dfatrue.Minus(dfatrue.Minus(dfadiff1, solver).Intersect(dfatrue.Minus(dfadiff2, solver), solver), solver).Determinize(solver).Minimize(solver);

            //Use smallest of |dfa1| and complement of |dfa1| for cardinality base
            return(GetDFARatio(dfa1.Determinize(solver).Minimize(solver), dfadiff, al, solver, solutionDensity > 0.5));
        }
        public static bool canCollapseStates(Automaton <BDD> nfa, int state1, int state2, CharSetSolver solver, Pair <IEnumerable <string>, IEnumerable <string> > tests, HashSet <char> al)
        {
            var density = DFADensity.GetDFADensity(nfa, al, solver);

            // collapses state2 to state1
            List <Move <BDD> > newMoves = new List <Move <BDD> >();

            foreach (var move in nfa.GetMoves())
            {
                var newSource = move.SourceState;
                var newTarget = move.TargetState;

                if (newSource == state2)
                {
                    newSource = state1;
                }
                if (newTarget == state2)
                {
                    newTarget = state1;
                }

                newMoves.Add(new Move <BDD>(newSource, newTarget, move.Label));
            }

            // replace state2 with state1 if initial state
            // no need to remove state2 from final state list, as it is unreachable
            int newInitialState = nfa.InitialState;

            if (nfa.InitialState == state2)
            {
                newInitialState = state1;
            }

            //makes new Nfa and returns collapse state edit if are equiv
            var newNfa = Automaton <BDD> .Create(newInitialState, nfa.GetFinalStates(), newMoves);

            if (DFAUtilities.ApproximateMNEquivalent(tests, density, newNfa, al, solver))
            {
                return(nfa.IsEquivalentWith(newNfa, solver));
            }
            return(false);
        }
        /// <summary>
        /// create new instance of NFAEdit distance and assigns a number to each character
        /// </summary>
        /// <param name="nfa1"></param>
        /// <param name="nfa2"></param>
        /// <param name="al"></param>
        /// <param name="solver"></param>
        /// <param name="timeout"></param>
        public NFAEditDistanceProvider(Automaton <BDD> nfa1,
                                       HashSet <char> al, CharSetSolver solver, long timeout)
        {
            this.nfa1 = nfa1;

            this.al          = al;
            this.solver      = solver;
            this.timeout     = timeout;
            this.alphabetMap = new Dictionary <char, int>();
            int index = 0;

            foreach (var c in al)
            {
                this.alphabetMap[c] = index;
                index++;
            }
            this.sw    = new Stopwatch();
            this.tests = NFAUtilities.MyHillTestGeneration(al, nfa1.Determinize(solver), solver);
            var dfa1 = nfa1.RemoveEpsilons(solver.MkOr).Determinize(solver).Minimize(solver);

            this.nfa1density = DFADensity.GetDFADensity(dfa1, al, solver);
        }
Beispiel #4
0
        /// <summary>
        /// Computes the grade for attempt using all the possible metrics
        /// </summary>
        /// <param name="solutionNFA">correct nfa</param>
        /// <param name="attemptNFA">nfa to be graded</param>
        /// <param name="alpahbet">input alphabet</param>
        /// <param name="solver">SMT solver for char set</param>
        /// <param name="timeout">timeout for the PDL enumeration (suggested > 1000)</param>
        /// <param name="maxGrade">Max grade for the homework</param>
        /// <param name="level">Feedback level</param>
        /// <returns>Grade for nfa2</returns>
        public static Pair <int, IEnumerable <NFAFeedback> > GetGrade(
            Automaton <BDD> solutionNFA, Automaton <BDD> attemptNFA, HashSet <char> alphabet,
            CharSetSolver solver, long timeout, int maxGrade, FeedbackLevel level)
        {
            var feedbacks          = new List <NFAFeedback>();
            int deadStateDeduction = 0;
            int tooBigDeduction    = 0;
            int incorrectDeduction = 0;

            // Remove at most a percentage of max grade when NFA is big
            double maxDeductionForTooBig     = ((double)maxGrade * 0.3);
            double maxDeductionForDeadStates = ((double)maxGrade * 0.1);


            double solutionStateCount      = solutionNFA.StateCount;
            double attemptStateCount       = attemptNFA.StateCount;
            double solutionTransCount      = solutionNFA.MoveCount;
            double attemptTransCount       = attemptNFA.MoveCount;
            NFAEditDistanceProvider nfaedp = new NFAEditDistanceProvider(solutionNFA, alphabet, solver, timeout);

            //Check if using epsilon and nondeterminism
            if (solutionNFA.IsEpsilonFree)
            {
                solutionNFA.CheckDeterminism(solver);
            }
            if (attemptNFA.IsEpsilonFree)
            {
                attemptNFA.CheckDeterminism(solver);
            }

            bool shouldUseEpsilon = !solutionNFA.IsEpsilonFree && attemptNFA.IsEpsilonFree;
            bool shouldUseNonDet  = !shouldUseEpsilon && !solutionNFA.isDeterministic && attemptNFA.isDeterministic;

            //Check if solution has dead states and remove if it does
            var statesBeforeDeadStatesElimination = attemptNFA.StateCount;

            attemptNFA.EliminateDeadStates();
            var solutionHasDeadStates = attemptNFA.StateCount < statesBeforeDeadStatesElimination;

            //Start checking equiv
            bool areEquivalent = solutionNFA.IsEquivalentWith(attemptNFA, solver);

            if (areEquivalent)
            {
                // prompt nfa is correct
                feedbacks.Insert(0, new NFAStringFeedback(level, alphabet, solver, "Your NFA accepts the CORRECT language."));

                #region Check number of states and decrease grade if too big
                int stateDiff = (int)(attemptStateCount - solutionStateCount);
                int transDiff = (int)(attemptTransCount - solutionTransCount);

                //If is not minimal apply deduction and compute edit
                if (stateDiff > 0 || transDiff > 0)
                {
                    #region Try to collapse for feedback
                    // Try to find a way to collaps states or remove states and transitions to make the NFA smaller
                    NFAEditScript collapseScript = null;
                    var           edit           = nfaedp.NFACollapseSearch(attemptNFA);
                    if (edit != null)
                    {
                        collapseScript = new NFAEditScript();
                        collapseScript.script.Insert(0, edit);
                    }
                    feedbacks.Add(new NFANotMinimalFeedback(level, alphabet, stateDiff, transDiff, collapseScript, solver));
                    #endregion

                    #region Compute tooBigDeduction
                    if (stateDiff > 0)
                    {
                        // ((att/sol)^2)-1
                        var stateRatio       = attemptStateCount / solutionStateCount;
                        var stateRatioSqM1   = Math.Pow(stateRatio, 2) - 1;
                        var sclaedStateRatio = stateRatioSqM1 * maxDeductionForTooBig / 2;

                        tooBigDeduction = (int)Math.Round(Math.Min(sclaedStateRatio, maxDeductionForTooBig));
                    }
                    else
                    {
                        if (transDiff > 0)
                        {
                            // ((att/sol)^2)-1
                            var transRatio       = attemptTransCount / solutionTransCount;
                            var transRatioSqM1   = Math.Pow(transRatio, 2) - 1;
                            var sclaedTransRatio = transRatioSqM1 * maxDeductionForTooBig / 2;

                            tooBigDeduction = (int)Math.Round(Math.Min(sclaedTransRatio, maxDeductionForTooBig));
                        }
                    }
                    //Make sure deduction is positive
                    tooBigDeduction = Math.Max(tooBigDeduction, 0);
                    #endregion
                }
                #endregion
            }
            else
            {
                // prompt nfa is incorrect
                feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "Your NFA does NOT accept the correct langauge."));

                //inequivalent, try using grading metrics and based on winning metric give feedback
                int remainingGrade = maxGrade - tooBigDeduction;

                #region metric computation
                //compute deterministic versions
                var solutionNFAdet = solutionNFA.RemoveEpsilons(solver.MkOr).Determinize(solver).MakeTotal(solver).Minimize(solver);
                var attemptNFAdet  = attemptNFA.RemoveEpsilons(solver.MkOr).Determinize(solver).MakeTotal(solver).Minimize(solver);
                solutionNFAdet.EliminateDeadStates();
                attemptNFAdet.EliminateDeadStates();

                //compute density
                double densityRatio = DFADensity.GetDFADifferenceRatio(solutionNFAdet, attemptNFAdet, alphabet, solver);

                //compute edit distance
                double nfaED      = 2;
                var    editScript = nfaedp.GetNFAOptimalEdit(attemptNFA);

                if (editScript != null)
                {
                    nfaED = ((double)(editScript.GetCost())) / ((double)((solutionNFA.StateCount + 1) * alphabet.Count));
                }
                #endregion

                #region metrics scaling
                var scalingSquareDensity = 1; var multv2 = 0.5;
                var scalingSquareDFAED = 1.03;

                var scaledDensityRatio = (scalingSquareDensity + (multv2 * densityRatio)) * (scalingSquareDensity + (multv2 * densityRatio)) - scalingSquareDensity * scalingSquareDensity;
                var scaledNfaED        = (scalingSquareDFAED + nfaED) * (scalingSquareDFAED + nfaED) - scalingSquareDFAED * scalingSquareDFAED;

                //If the edit script was not computed make sure it loses.
                if (editScript == null)
                {
                    scaledNfaED = Double.MaxValue;
                }

                //Select dominating Feedback based on grade
                double unscaledGrade = Math.Min(scaledDensityRatio, scaledNfaED);
                var    dfaedwins     = scaledNfaED <= scaledDensityRatio;
                var    densitywins   = scaledDensityRatio <= scaledNfaED;

                incorrectDeduction = (int)Math.Round(unscaledGrade * (double)(maxGrade));
                #endregion


                //If edit distance search works, provides feedback based upon result
                //Otherwise, gives counterexample feedback
                if (level != FeedbackLevel.Minimal)
                {
                    if (dfaedwins)
                    {
                        feedbacks.Add(new NFAEDFeedback(solutionNFA, attemptNFA, level, alphabet, editScript, solver));
                    }
                    else
                    {
                        feedbacks.Add(new NFACounterexampleFeedback(level, alphabet, solutionNFAdet, attemptNFAdet, solver));
                    }
                }
            }

            // Feedback related to nondeterminism and epislon
            if (shouldUseEpsilon)
            {
                feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "You should try using epsilon transitions."));
            }
            if (shouldUseNonDet)
            {
                feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "You should try using nondeterminism."));
            }

            // Deduct points and prompt feedback is solution has dead states
            if (solutionHasDeadStates)
            {
                deadStateDeduction = (int)maxDeductionForDeadStates;
                feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "Your NFA has some dead states."));
            }

            //Grade computation

            //changed to be binary, because we did not tell them of the deductions
            //int grade = Math.Max(maxGrade - deadStateDeduction - tooBigDeduction - incorrectDeduction, 0);
            //int grade = areEquivalent ? maxGrade : 0;

            // This requires further testing, but appears to grade ok, 0 points for empty automaton and full points for something in between
            int grade = Math.Max(maxGrade - incorrectDeduction, 0);

            return(new Pair <int, IEnumerable <NFAFeedback> >(grade, feedbacks));
        }
Beispiel #5
0
        /// <summary>
        /// Computes the grade for attempt using all the possible metrics
        /// </summary>
        /// <param name="dfaGoal">minimal correct dfa</param>
        /// <param name="dfaAttempt">dfa to be graded</param>
        /// <param name="al">input alphabet</param>
        /// <param name="solver">SMT solver for char set</param>
        /// <param name="timeout">timeout for the PDL enumeration (suggested > 1000)</param>
        /// <param name="maxGrade">Max grade for the homework</param>
        /// <param name="enableDFAED">true to enable DFA edit distance</param>
        /// <param name="enablePDLED">true to enable PDL edit distance</param>
        /// <param name="enableDensity">true to enable density distance</param>
        /// <returns>Grade for dfa2</returns>
        public static Pair <int, IEnumerable <DFAFeedback> > GetGrade(
            Automaton <BDD> dfaGoal, Automaton <BDD> dfaAttempt, HashSet <char> al,
            CharSetSolver solver, long timeout,
            int maxGrade, FeedbackLevel level,
            bool enableDFAED, bool enablePDLED, bool enableDensity)
        {
            PDLEnumerator pdlEnumerator = new PDLEnumerator();

            var feedList = new List <DFAFeedback>();

            DFAFeedback defaultFeedback = new StringFeedback(level, StringFeedbackType.Wrong, al, solver);

            #region Accessory and initial vars
            //Compute minimized version of DFAs
            var dfaGoalMin    = dfaGoal.Determinize(solver).Minimize(solver);
            var dfaAttemptMin = dfaAttempt.Determinize(solver).Minimize(solver);

            //Initialize distances at high values in case they are not used
            // they only produce positive grade if between 0 and 1
            double pdlEditDistanceScaled = 2;
            double densityRatio          = 2;
            double dfaED = 2;
            #endregion

            #region deductions on the grade based on the size of the dfa
            //Deduction if DFA is smaller than it should be: used only for PDL ed and for density
            var smallerDFADeduction = 0.2 * Math.Sqrt(
                Math.Max(0.0, dfaGoalMin.StateCount - dfaAttemptMin.StateCount) /
                ((double)dfaGoalMin.StateCount));
            #endregion

            #region check whether the attempt is equivalent to the solution
            if (dfaGoal.IsEquivalentWith(dfaAttempt, solver))
            {
                Console.WriteLine("Correct");
                feedList.Add(new StringFeedback(level, StringFeedbackType.Correct, al, solver));
                return(new Pair <int, IEnumerable <DFAFeedback> >(maxGrade, feedList));
            }
            #endregion

            #region metrics computation
            Stopwatch swPDLed = new Stopwatch();
            swPDLed.Start();
            #region PDL edit distance
            Transformation feedbackTransformation = null;
            if (enablePDLED)
            {
                var trpair = PDLEditDistance.GetMinimalFormulaEditDistanceTransformation(dfaGoalMin, dfaAttemptMin, al, solver, timeout, pdlEnumerator);

                if (trpair != null)
                {
                    var transformationGrade = trpair.First;
                    feedbackTransformation = trpair.Second;
                    var scaling = 1.0;
                    pdlEditDistanceScaled = transformationGrade.totalCost / (transformationGrade.minSizeForTreeA * scaling) + smallerDFADeduction;
                }
            }
            #endregion
            swPDLed.Stop();

            Stopwatch swDensity = new Stopwatch();
            swDensity.Start();
            #region density distance
            if (enableDensity)
            {
                densityRatio  = DFADensity.GetDFADifferenceRatio(dfaGoalMin, dfaAttemptMin, al, solver);
                densityRatio += smallerDFADeduction;
            }
            #endregion
            swDensity.Stop();

            Stopwatch swDFAed = new Stopwatch();
            swDFAed.Start();
            #region DFA edit distance
            DFAEditScript dfaEditScript = null;
            if (enableDFAED)
            {
                //limit the depth of the DFA edit distance search
                var maxMoves = Math.Max(1, 6 - (int)Math.Sqrt(dfaAttempt.MoveCount + dfaAttempt.StateCount));
                dfaEditScript = DFAEditDistance.GetDFAOptimalEdit(dfaGoal, dfaAttempt, al, solver, timeout, new StringBuilder());

                if (dfaEditScript != null)
                {
                    dfaED = ((double)(dfaEditScript.GetCost())) / ((double)((dfaGoalMin.StateCount + 1) * al.Count));
                }
            }
            #endregion
            swDFAed.Stop();

            #endregion

            #region metrics scaling
            var scalingSquarePDLED = 1.005;
            var scalingSquareDensity = 1; var multv2 = 0.5;
            var scalingSquareDFAED = 1.03;

            var scaledPdlED        = (0.9 * (scalingSquarePDLED + pdlEditDistanceScaled) * (scalingSquarePDLED + pdlEditDistanceScaled)) - scalingSquarePDLED * scalingSquarePDLED;
            var scaledDensityRatio = (scalingSquareDensity + (multv2 * densityRatio)) * (scalingSquareDensity + (multv2 * densityRatio)) - scalingSquareDensity * scalingSquareDensity;
            var scaledDfaED        = (scalingSquareDFAED + dfaED) * (scalingSquareDFAED + dfaED) - scalingSquareDFAED * scalingSquareDFAED;


            //Select dominating Feedback based on grade
            double unscaledGrade = Math.Min(Math.Min(scaledPdlED, scaledDensityRatio), scaledDfaED);
            var    pdledwins     = scaledPdlED <= Math.Min(scaledDensityRatio, scaledDfaED);
            var    dfaedwins     = scaledDfaED <= Math.Min(scaledDensityRatio, scaledPdlED);
            var    densitywins   = scaledDensityRatio <= Math.Min(scaledDfaED, scaledPdlED);
            #endregion

            #region Feedback Selection
            if (pdledwins && feedbackTransformation != null && feedbackTransformation.pdlB.GetFormulaSize() < 10)
            {
                feedList.Add(new PDLEDFeedback(level, al, feedbackTransformation, scaledPdlED, solver));
            }

            if ((dfaedwins || feedList.Count == 0) && dfaEditScript != null && !dfaEditScript.IsComplex())
            {
                feedList = new List <DFAFeedback>();
                feedList.Add(new DFAEDFeedback(dfaGoal, dfaAttempt, level, al, dfaEditScript, scaledDfaED, solver));
            }

            if (densitywins || feedList.Count == 0)
            {
                feedList = new List <DFAFeedback>();
                feedList.Add(new DensityFeedback(level, al, dfaGoal, dfaAttempt, scaledDensityRatio, solver));
            }

            if (feedList.Count == 0)
            {
                Console.WriteLine("Why no feedback!!");
                feedList.Add(defaultFeedback);
            }
            #endregion

            #region normalize grade
            var scaledGrade = maxGrade - (int)Math.Round(unscaledGrade * (double)(maxGrade));
            //If rounding yields maxgrade deduct 1 point by default
            if (scaledGrade == maxGrade)
            {
                scaledGrade = maxGrade - 1;
            }

            //Remove possible deduction
            scaledGrade = scaledGrade < 0 ? 0 : scaledGrade;
            return(new Pair <int, IEnumerable <DFAFeedback> >(scaledGrade, feedList));

            #endregion
        }
Beispiel #6
0
        /// <summary>
        /// Finds min edit distance script between DFAs if operation
        /// takes less than timeout ms
        /// </summary>
        /// <param name="dfa1"></param>
        /// <param name="dfa2"></param>
        /// <param name="al"></param>
        /// <param name="solver"></param>
        /// <param name="timeout"></param>
        /// <param name="sb"></param>
        /// <returns></returns>
        public static DFAEditScript GetDFAOptimalEdit( // copy
            Automaton <BDD> dfa1, Automaton <BDD> dfa2,
            HashSet <char> al, CharSetSolver solver, long timeout,
            StringBuilder sb)
        {
            //Contract.Assert(dfa1.IsDeterministic);
            //Contract.Assert(dfa2.IsDeterministic);

            DFAEditScript editScript = new DFAEditScript();

            #region Add states to dfa2 to make it at least as dfa1
            BDD fullAlphabetCondition = BDDOf(al, solver);

            //Normalize the DFA giving only names from 0 to |States|-1
            var normDfaPair   = DFAUtilities.normalizeDFA(dfa2);
            var dfa2augmented = normDfaPair.First;
            //solver.SaveAsDot(dfa2augmented, "aaaa");
            var stateNamesMapping = normDfaPair.Second;

            //Add states to make dfa2 have the |dfa2.States|>= |dfa1.States|
            var newMoves = new List <Move <BDD> >(dfa2augmented.GetMoves());

            for (int i = 1; i <= dfa1.StateCount - dfa2augmented.StateCount; i++)
            {
                int newStateName = dfa2augmented.MaxState + i;
                //Pick the next available name to be added
                stateNamesMapping[newStateName] = dfa2.MaxState + i;
                //save the operation in the script
                editScript.script.Insert(0, new DFAAddState(dfa2.MaxState + i));
                newMoves.Add(new Move <BDD>(newStateName, newStateName, fullAlphabetCondition));
                newStateName++;
            }
            //Create the new DFA with the added states
            dfa2augmented = Automaton <BDD> .Create(dfa2augmented.InitialState, dfa2augmented.GetFinalStates().ToList(), newMoves);

            #endregion

            int maxScore      = (dfa1.StateCount + dfa2augmented.StateCount) * (al.Count + 1);
            int oldScirptSize = editScript.script.Count;

            //Start with the internal script equals to null, at the end bestScript.Script will contain the best script
            DFAEditScript bestScript = new DFAEditScript();
            bestScript.script = null;
            Stopwatch sw = new Stopwatch();
            sw.Start();

            // Iteratively check if there exists an edit of a given depth
            for (int depth = 1; true; depth++)
            {
                var editList = new List <DFAEdit>();
                if (GetDFAEditScriptTimeout(
                        dfa1, dfa2augmented, al, solver,
                        new List <long>(), editScript.script,
                        depth, timeout, sw, DFAUtilities.MyHillTestGeneration(al, dfa1, solver),
                        DFADensity.GetDFADensity(dfa1, al, solver),
                        editScript.GetCost(), bestScript, stateNamesMapping))
                {
                    // if hits timeout break and return null
                    break;
                }
                if (bestScript.script != null)
                {
                    bestScript.script.Reverse();
                    sw.Stop();
                    return(bestScript);
                }
            }

            sw.Stop();
            return(null);
        }