public DFAEDFeedback(Automaton <BDD> dfaGoal, Automaton <BDD> dfaAttempt, FeedbackLevel level, HashSet <char> alphabet, DFAEditScript script, double utility, CharSetSolver solver)
            : base(level, alphabet, utility, solver)
        {
            var positiveDifference = dfaGoal.Minus(dfaAttempt, solver).Determinize(solver).Minimize(solver);
            var negativeDifference = dfaAttempt.Minus(dfaGoal, solver).Determinize(solver).Minimize(solver);

            this.counterexample = DFAUtilities.GenerateShortTerm(positiveDifference.IsEmpty ? negativeDifference : positiveDifference, solver);
            this.type           = FeedbackType.DFAED;
            this.script         = script;
        }
 public DFAEDFeedback(Automaton<BDD> dfaGoal, Automaton<BDD> dfaAttempt, FeedbackLevel level, HashSet<char> alphabet, DFAEditScript script, double utility, CharSetSolver solver)
     : base(level, alphabet, utility, solver)
 {
     var positiveDifference = dfaGoal.Minus(dfaAttempt, solver).Determinize(solver).Minimize(solver);
     var negativeDifference = dfaAttempt.Minus(dfaGoal, solver).Determinize(solver).Minimize(solver);
     this.counterexample = DFAUtilities.GenerateShortTerm(positiveDifference.IsEmpty ? negativeDifference : positiveDifference, solver);
     this.type = FeedbackType.DFAED;
     this.script = script;
 }
Esempio n. 3
0
        /// <summary>
        /// Computes the grade for attempt using all the possible metrics
        /// </summary>
        /// <param name="dfaGoal">minimal correct dfa</param>
        /// <param name="dfaAttempt">dfa to be graded</param>
        /// <param name="al">input alphabet</param>
        /// <param name="solver">SMT solver for char set</param>
        /// <param name="timeout">timeout for the PDL enumeration (suggested > 1000)</param>
        /// <param name="maxGrade">Max grade for the homework</param>
        /// <param name="enableDFAED">true to enable DFA edit distance</param>
        /// <param name="enablePDLED">true to enable PDL edit distance</param>
        /// <param name="enableDensity">true to enable density distance</param>
        /// <returns>Grade for dfa2</returns>
        public static Pair <int, IEnumerable <DFAFeedback> > GetGrade(
            Automaton <BDD> dfaGoal, Automaton <BDD> dfaAttempt, HashSet <char> al,
            CharSetSolver solver, long timeout,
            int maxGrade, FeedbackLevel level,
            bool enableDFAED, bool enablePDLED, bool enableDensity)
        {
            PDLEnumerator pdlEnumerator = new PDLEnumerator();

            var feedList = new List <DFAFeedback>();

            DFAFeedback defaultFeedback = new StringFeedback(level, StringFeedbackType.Wrong, al, solver);

            #region Accessory and initial vars
            //Compute minimized version of DFAs
            var dfaGoalMin    = dfaGoal.Determinize(solver).Minimize(solver);
            var dfaAttemptMin = dfaAttempt.Determinize(solver).Minimize(solver);

            //Initialize distances at high values in case they are not used
            // they only produce positive grade if between 0 and 1
            double pdlEditDistanceScaled = 2;
            double densityRatio          = 2;
            double dfaED = 2;
            #endregion

            #region deductions on the grade based on the size of the dfa
            //Deduction if DFA is smaller than it should be: used only for PDL ed and for density
            var smallerDFADeduction = 0.2 * Math.Sqrt(
                Math.Max(0.0, dfaGoalMin.StateCount - dfaAttemptMin.StateCount) /
                ((double)dfaGoalMin.StateCount));
            #endregion

            #region check whether the attempt is equivalent to the solution
            if (dfaGoal.IsEquivalentWith(dfaAttempt, solver))
            {
                Console.WriteLine("Correct");
                feedList.Add(new StringFeedback(level, StringFeedbackType.Correct, al, solver));
                return(new Pair <int, IEnumerable <DFAFeedback> >(maxGrade, feedList));
            }
            #endregion

            #region metrics computation
            Stopwatch swPDLed = new Stopwatch();
            swPDLed.Start();
            #region PDL edit distance
            Transformation feedbackTransformation = null;
            if (enablePDLED)
            {
                var trpair = PDLEditDistance.GetMinimalFormulaEditDistanceTransformation(dfaGoalMin, dfaAttemptMin, al, solver, timeout, pdlEnumerator);

                if (trpair != null)
                {
                    var transformationGrade = trpair.First;
                    feedbackTransformation = trpair.Second;
                    var scaling = 1.0;
                    pdlEditDistanceScaled = transformationGrade.totalCost / (transformationGrade.minSizeForTreeA * scaling) + smallerDFADeduction;
                }
            }
            #endregion
            swPDLed.Stop();

            Stopwatch swDensity = new Stopwatch();
            swDensity.Start();
            #region density distance
            if (enableDensity)
            {
                densityRatio  = DFADensity.GetDFADifferenceRatio(dfaGoalMin, dfaAttemptMin, al, solver);
                densityRatio += smallerDFADeduction;
            }
            #endregion
            swDensity.Stop();

            Stopwatch swDFAed = new Stopwatch();
            swDFAed.Start();
            #region DFA edit distance
            DFAEditScript dfaEditScript = null;
            if (enableDFAED)
            {
                //limit the depth of the DFA edit distance search
                var maxMoves = Math.Max(1, 6 - (int)Math.Sqrt(dfaAttempt.MoveCount + dfaAttempt.StateCount));
                dfaEditScript = DFAEditDistance.GetDFAOptimalEdit(dfaGoal, dfaAttempt, al, solver, timeout, new StringBuilder());

                if (dfaEditScript != null)
                {
                    dfaED = ((double)(dfaEditScript.GetCost())) / ((double)((dfaGoalMin.StateCount + 1) * al.Count));
                }
            }
            #endregion
            swDFAed.Stop();

            #endregion

            #region metrics scaling
            var scalingSquarePDLED = 1.005;
            var scalingSquareDensity = 1; var multv2 = 0.5;
            var scalingSquareDFAED = 1.03;

            var scaledPdlED        = (0.9 * (scalingSquarePDLED + pdlEditDistanceScaled) * (scalingSquarePDLED + pdlEditDistanceScaled)) - scalingSquarePDLED * scalingSquarePDLED;
            var scaledDensityRatio = (scalingSquareDensity + (multv2 * densityRatio)) * (scalingSquareDensity + (multv2 * densityRatio)) - scalingSquareDensity * scalingSquareDensity;
            var scaledDfaED        = (scalingSquareDFAED + dfaED) * (scalingSquareDFAED + dfaED) - scalingSquareDFAED * scalingSquareDFAED;


            //Select dominating Feedback based on grade
            double unscaledGrade = Math.Min(Math.Min(scaledPdlED, scaledDensityRatio), scaledDfaED);
            var    pdledwins     = scaledPdlED <= Math.Min(scaledDensityRatio, scaledDfaED);
            var    dfaedwins     = scaledDfaED <= Math.Min(scaledDensityRatio, scaledPdlED);
            var    densitywins   = scaledDensityRatio <= Math.Min(scaledDfaED, scaledPdlED);
            #endregion

            #region Feedback Selection
            if (pdledwins && feedbackTransformation != null && feedbackTransformation.pdlB.GetFormulaSize() < 10)
            {
                feedList.Add(new PDLEDFeedback(level, al, feedbackTransformation, scaledPdlED, solver));
            }

            if ((dfaedwins || feedList.Count == 0) && dfaEditScript != null && !dfaEditScript.IsComplex())
            {
                feedList = new List <DFAFeedback>();
                feedList.Add(new DFAEDFeedback(dfaGoal, dfaAttempt, level, al, dfaEditScript, scaledDfaED, solver));
            }

            if (densitywins || feedList.Count == 0)
            {
                feedList = new List <DFAFeedback>();
                feedList.Add(new DensityFeedback(level, al, dfaGoal, dfaAttempt, scaledDensityRatio, solver));
            }

            if (feedList.Count == 0)
            {
                Console.WriteLine("Why no feedback!!");
                feedList.Add(defaultFeedback);
            }
            #endregion

            #region normalize grade
            var scaledGrade = maxGrade - (int)Math.Round(unscaledGrade * (double)(maxGrade));
            //If rounding yields maxgrade deduct 1 point by default
            if (scaledGrade == maxGrade)
            {
                scaledGrade = maxGrade - 1;
            }

            //Remove possible deduction
            scaledGrade = scaledGrade < 0 ? 0 : scaledGrade;
            return(new Pair <int, IEnumerable <DFAFeedback> >(scaledGrade, feedList));

            #endregion
        }
        /// <summary>
        /// Finds min edit distance script between DFAs if operation
        /// takes less than timeout ms
        /// </summary>
        /// <param name="dfa1"></param>
        /// <param name="dfa2"></param>
        /// <param name="al"></param>
        /// <param name="solver"></param>
        /// <param name="timeout"></param>
        /// <param name="sb"></param>
        /// <returns></returns>
        public static DFAEditScript GetDFAOptimalEdit( // copy
            Automaton<BDD> dfa1, Automaton<BDD> dfa2,
            HashSet<char> al, CharSetSolver solver, long timeout,
            StringBuilder sb)
        {
            //Contract.Assert(dfa1.IsDeterministic);
            //Contract.Assert(dfa2.IsDeterministic);

            DFAEditScript editScript = new DFAEditScript();

            #region Add states to dfa2 to make it at least as dfa1
            BDD fullAlphabetCondition = BDDOf(al, solver);

            //Normalize the DFA giving only names from 0 to |States|-1
            var normDfaPair = DFAUtilities.normalizeDFA(dfa2);
            var dfa2augmented = normDfaPair.First;
            //solver.SaveAsDot(dfa2augmented, "aaaa");
            var stateNamesMapping = normDfaPair.Second;

            //Add states to make dfa2 have the |dfa2.States|>= |dfa1.States|
            var newMoves = new List<Move<BDD>>(dfa2augmented.GetMoves());

            for (int i = 1; i <= dfa1.StateCount - dfa2augmented.StateCount; i++)
            {
                int newStateName = dfa2augmented.MaxState + i;
                //Pick the next available name to be added
                stateNamesMapping[newStateName] = dfa2.MaxState + i;
                //save the operation in the script
                editScript.script.Insert(0, new DFAAddState(dfa2.MaxState + i));
                newMoves.Add(new Move<BDD>(newStateName, newStateName, fullAlphabetCondition));
                newStateName++;
            }
            //Create the new DFA with the added states
            dfa2augmented = Automaton<BDD>.Create(dfa2augmented.InitialState, dfa2augmented.GetFinalStates().ToList(), newMoves);
            #endregion

            int maxScore = (dfa1.StateCount + dfa2augmented.StateCount) * (al.Count + 1);
            int oldScirptSize = editScript.script.Count;

            //Start with the internal script equals to null, at the end bestScript.Script will contain the best script
            DFAEditScript bestScript = new DFAEditScript();
            bestScript.script = null;
            Stopwatch sw = new Stopwatch();
            sw.Start();

            // Iteratively check if there exists an edit of a given depth            
            for (int depth = 1; true; depth++)
            {
                var editList = new List<DFAEdit>();
                if (GetDFAEditScriptTimeout(
                    dfa1, dfa2augmented, al, solver,
                    new List<long>(), editScript.script,
                    depth, timeout, sw, DFAUtilities.MyHillTestGeneration(al, dfa1, solver),
                    DFADensity.GetDFADensity(dfa1, al, solver),
                    editScript.GetCost(), bestScript, stateNamesMapping))
                {
                    // if hits timeout break and return null
                    break;
                }
                if (bestScript.script != null)
                {
                    bestScript.script.Reverse();
                    sw.Stop();
                    return bestScript;
                }
            }

            sw.Stop();
            return null;
        }
        // looks for an edit at depth "depth" 
        // returns false and null in bestScript if no edit is found at depth "depth"
        // returns false and not null in bestScript if found
        // returns true if timeout
        internal static bool GetDFAEditScriptTimeout(
            Automaton<BDD> dfa1, Automaton<BDD> dfa2,
            HashSet<char> al, CharSetSolver solver,
            List<long> editScriptHash, List<DFAEdit> editList,
            int depth, long timeout, Stopwatch sw,
            Pair<IEnumerable<string>, IEnumerable<string>> tests,
            double dfa1density,
            int totalCost,
            DFAEditScript bestScript, Dictionary<int, int> stateNamesMapping)
        {
            // check timer
            if (sw.ElapsedMilliseconds > timeout)
                return true;

            //Compute worst case distance, call finalScript with this value?
            int dist = (dfa1.StateCount + dfa2.StateCount) * (al.Count + 1);

            //Stop if no more moves left
            if (depth == 0)
            {
                if (DFAUtilities.ApproximateMNEquivalent(tests, dfa1density, dfa2, al, solver) && dfa2.IsEquivalentWith(dfa1, solver))
                    //check if totalCost < finalScript cost and replace if needed
                    if (bestScript.script == null || totalCost < bestScript.GetCost())
                        bestScript.script = ObjectCopier.Clone<List<DFAEdit>>(editList);
                return false;
            }

            DFAEdit edit = null;

            #region Flip one move target state
            foreach (var move in dfa2.GetMoves())
            {
                //Creaty copy of the moves without current move
                var movesWithoutCurrMove = dfa2.GetMoves().ToList();
                movesWithoutCurrMove.Remove(move);

                //Redirect every ch belonging to move condition
                foreach (var c in solver.GenerateAllCharacters(move.Label, false))
                {
                    long hash = IntegerUtil.PairToInt(move.SourceState, c - 97) + dfa2.StateCount;

                    if (CanAdd(hash, editScriptHash))
                    {
                        editScriptHash.Insert(0, hash);

                        //Local copy of moves
                        var newMoves = movesWithoutCurrMove.ToList();
                        var newMoveCondition = solver.MkCharConstraint(false, c);

                        #region Remove ch from current move
                        var andCond = solver.MkAnd(move.Label, solver.MkNot(newMoveCondition));
                        //add back move without ch iff satisfiable
                        if (solver.IsSatisfiable(andCond))
                            newMoves.Add(new Move<BDD>(move.SourceState, move.TargetState, andCond));
                        #endregion

                        #region Redirect c to a different state
                        foreach (var state in dfa2.States)
                            if (state != move.TargetState)
                            {
                                var newMovesComplete = newMoves.ToList();
                                newMovesComplete.Add(new Move<BDD>(move.SourceState, state, newMoveCondition));
                                var dfa2new = Automaton<BDD>.Create(dfa2.InitialState, dfa2.GetFinalStates(), newMovesComplete);

                                edit = new DFAEditMove(stateNamesMapping[move.SourceState], stateNamesMapping[state], c);
                                editList.Insert(0, edit);
                                if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
                                    return true;
                                editList.RemoveAt(0);
                            }
                        #endregion

                        editScriptHash.RemoveAt(0);

                    }
                }
            }
            #endregion

            #region Flip one state from fin to non fin
            foreach (var state in dfa2.States)
            {
                if (CanAdd(state, editScriptHash))
                {
                    //flip its final non final status
                    editScriptHash.Insert(0, state);

                    var newFinalStates = new HashSet<int>(dfa2.GetFinalStates());
                    Automaton<BDD> dfa2new = null;
                    if (dfa2.GetFinalStates().Contains(state))
                    {
                        edit = new DFAEditState(stateNamesMapping[state], false);
                        editList.Insert(0, edit);
                        newFinalStates.Remove(state);
                        dfa2new = Automaton<BDD>.Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
                    }
                    else
                    {
                        edit = new DFAEditState(stateNamesMapping[state], true);
                        editList.Insert(0, edit);
                        newFinalStates.Add(state);
                        dfa2new = Automaton<BDD>.Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
                    }

                    if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
                        return true;

                    editScriptHash.RemoveAt(0);
                    editList.RemoveAt(0);
                }
            }
            #endregion

            return false;
        }
Esempio n. 6
0
        /// <summary>
        /// Finds min edit distance script between DFAs if operation
        /// takes less than timeout ms
        /// </summary>
        /// <param name="dfa1"></param>
        /// <param name="dfa2"></param>
        /// <param name="al"></param>
        /// <param name="solver"></param>
        /// <param name="timeout"></param>
        /// <param name="sb"></param>
        /// <returns></returns>
        public static DFAEditScript GetDFAOptimalEdit( // copy
            Automaton <BDD> dfa1, Automaton <BDD> dfa2,
            HashSet <char> al, CharSetSolver solver, long timeout,
            StringBuilder sb)
        {
            //Contract.Assert(dfa1.IsDeterministic);
            //Contract.Assert(dfa2.IsDeterministic);

            DFAEditScript editScript = new DFAEditScript();

            #region Add states to dfa2 to make it at least as dfa1
            BDD fullAlphabetCondition = BDDOf(al, solver);

            //Normalize the DFA giving only names from 0 to |States|-1
            var normDfaPair   = DFAUtilities.normalizeDFA(dfa2);
            var dfa2augmented = normDfaPair.First;
            //solver.SaveAsDot(dfa2augmented, "aaaa");
            var stateNamesMapping = normDfaPair.Second;

            //Add states to make dfa2 have the |dfa2.States|>= |dfa1.States|
            var newMoves = new List <Move <BDD> >(dfa2augmented.GetMoves());

            for (int i = 1; i <= dfa1.StateCount - dfa2augmented.StateCount; i++)
            {
                int newStateName = dfa2augmented.MaxState + i;
                //Pick the next available name to be added
                stateNamesMapping[newStateName] = dfa2.MaxState + i;
                //save the operation in the script
                editScript.script.Insert(0, new DFAAddState(dfa2.MaxState + i));
                newMoves.Add(new Move <BDD>(newStateName, newStateName, fullAlphabetCondition));
                newStateName++;
            }
            //Create the new DFA with the added states
            dfa2augmented = Automaton <BDD> .Create(dfa2augmented.InitialState, dfa2augmented.GetFinalStates().ToList(), newMoves);

            #endregion

            int maxScore      = (dfa1.StateCount + dfa2augmented.StateCount) * (al.Count + 1);
            int oldScirptSize = editScript.script.Count;

            //Start with the internal script equals to null, at the end bestScript.Script will contain the best script
            DFAEditScript bestScript = new DFAEditScript();
            bestScript.script = null;
            Stopwatch sw = new Stopwatch();
            sw.Start();

            // Iteratively check if there exists an edit of a given depth
            for (int depth = 1; true; depth++)
            {
                var editList = new List <DFAEdit>();
                if (GetDFAEditScriptTimeout(
                        dfa1, dfa2augmented, al, solver,
                        new List <long>(), editScript.script,
                        depth, timeout, sw, DFAUtilities.MyHillTestGeneration(al, dfa1, solver),
                        DFADensity.GetDFADensity(dfa1, al, solver),
                        editScript.GetCost(), bestScript, stateNamesMapping))
                {
                    // if hits timeout break and return null
                    break;
                }
                if (bestScript.script != null)
                {
                    bestScript.script.Reverse();
                    sw.Stop();
                    return(bestScript);
                }
            }

            sw.Stop();
            return(null);
        }
Esempio n. 7
0
        // looks for an edit at depth "depth"
        // returns false and null in bestScript if no edit is found at depth "depth"
        // returns false and not null in bestScript if found
        // returns true if timeout
        internal static bool GetDFAEditScriptTimeout(
            Automaton <BDD> dfa1, Automaton <BDD> dfa2,
            HashSet <char> al, CharSetSolver solver,
            List <long> editScriptHash, List <DFAEdit> editList,
            int depth, long timeout, Stopwatch sw,
            Pair <IEnumerable <string>, IEnumerable <string> > tests,
            double dfa1density,
            int totalCost,
            DFAEditScript bestScript, Dictionary <int, int> stateNamesMapping)
        {
            // check timer
            if (sw.ElapsedMilliseconds > timeout)
            {
                return(true);
            }

            //Compute worst case distance, call finalScript with this value?
            int dist = (dfa1.StateCount + dfa2.StateCount) * (al.Count + 1);

            //Stop if no more moves left
            if (depth == 0)
            {
                if (DFAUtilities.ApproximateMNEquivalent(tests, dfa1density, dfa2, al, solver) && dfa2.IsEquivalentWith(dfa1, solver))
                {
                    //check if totalCost < finalScript cost and replace if needed
                    if (bestScript.script == null || totalCost < bestScript.GetCost())
                    {
                        bestScript.script = ObjectCopier.Clone <List <DFAEdit> >(editList);
                    }
                }
                return(false);
            }

            DFAEdit edit = null;

            #region Flip one move target state
            foreach (var move in dfa2.GetMoves())
            {
                //Creaty copy of the moves without current move
                var movesWithoutCurrMove = dfa2.GetMoves().ToList();
                movesWithoutCurrMove.Remove(move);

                //Redirect every ch belonging to move condition
                foreach (var c in solver.GenerateAllCharacters(move.Label, false))
                {
                    long hash = IntegerUtil.PairToInt(move.SourceState, c - 97) + dfa2.StateCount;

                    if (CanAdd(hash, editScriptHash))
                    {
                        editScriptHash.Insert(0, hash);

                        //Local copy of moves
                        var newMoves         = movesWithoutCurrMove.ToList();
                        var newMoveCondition = solver.MkCharConstraint(false, c);

                        #region Remove ch from current move
                        var andCond = solver.MkAnd(move.Label, solver.MkNot(newMoveCondition));
                        //add back move without ch iff satisfiable
                        if (solver.IsSatisfiable(andCond))
                        {
                            newMoves.Add(new Move <BDD>(move.SourceState, move.TargetState, andCond));
                        }
                        #endregion

                        #region Redirect c to a different state
                        foreach (var state in dfa2.States)
                        {
                            if (state != move.TargetState)
                            {
                                var newMovesComplete = newMoves.ToList();
                                newMovesComplete.Add(new Move <BDD>(move.SourceState, state, newMoveCondition));
                                var dfa2new = Automaton <BDD> .Create(dfa2.InitialState, dfa2.GetFinalStates(), newMovesComplete);

                                edit = new DFAEditMove(stateNamesMapping[move.SourceState], stateNamesMapping[state], c);
                                editList.Insert(0, edit);
                                if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
                                {
                                    return(true);
                                }
                                editList.RemoveAt(0);
                            }
                        }
                        #endregion

                        editScriptHash.RemoveAt(0);
                    }
                }
            }
            #endregion

            #region Flip one state from fin to non fin
            foreach (var state in dfa2.States)
            {
                if (CanAdd(state, editScriptHash))
                {
                    //flip its final non final status
                    editScriptHash.Insert(0, state);

                    var             newFinalStates = new HashSet <int>(dfa2.GetFinalStates());
                    Automaton <BDD> dfa2new        = null;
                    if (dfa2.GetFinalStates().Contains(state))
                    {
                        edit = new DFAEditState(stateNamesMapping[state], false);
                        editList.Insert(0, edit);
                        newFinalStates.Remove(state);
                        dfa2new = Automaton <BDD> .Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
                    }
                    else
                    {
                        edit = new DFAEditState(stateNamesMapping[state], true);
                        editList.Insert(0, edit);
                        newFinalStates.Add(state);
                        dfa2new = Automaton <BDD> .Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
                    }

                    if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
                    {
                        return(true);
                    }

                    editScriptHash.RemoveAt(0);
                    editList.RemoveAt(0);
                }
            }
            #endregion

            return(false);
        }