public DFAEDFeedback(Automaton <BDD> dfaGoal, Automaton <BDD> dfaAttempt, FeedbackLevel level, HashSet <char> alphabet, DFAEditScript script, double utility, CharSetSolver solver)
: base(level, alphabet, utility, solver)
{
var positiveDifference = dfaGoal.Minus(dfaAttempt, solver).Determinize(solver).Minimize(solver);
var negativeDifference = dfaAttempt.Minus(dfaGoal, solver).Determinize(solver).Minimize(solver);
this.counterexample = DFAUtilities.GenerateShortTerm(positiveDifference.IsEmpty ? negativeDifference : positiveDifference, solver);
this.type = FeedbackType.DFAED;
this.script = script;
}
public DFAEDFeedback(Automaton<BDD> dfaGoal, Automaton<BDD> dfaAttempt, FeedbackLevel level, HashSet<char> alphabet, DFAEditScript script, double utility, CharSetSolver solver)
: base(level, alphabet, utility, solver)
{
var positiveDifference = dfaGoal.Minus(dfaAttempt, solver).Determinize(solver).Minimize(solver);
var negativeDifference = dfaAttempt.Minus(dfaGoal, solver).Determinize(solver).Minimize(solver);
this.counterexample = DFAUtilities.GenerateShortTerm(positiveDifference.IsEmpty ? negativeDifference : positiveDifference, solver);
this.type = FeedbackType.DFAED;
this.script = script;
}
/// <summary>
/// Computes the grade for attempt using all the possible metrics
/// </summary>
/// <param name="dfaGoal">minimal correct dfa</param>
/// <param name="dfaAttempt">dfa to be graded</param>
/// <param name="al">input alphabet</param>
/// <param name="solver">SMT solver for char set</param>
/// <param name="timeout">timeout for the PDL enumeration (suggested > 1000)</param>
/// <param name="maxGrade">Max grade for the homework</param>
/// <param name="enableDFAED">true to enable DFA edit distance</param>
/// <param name="enablePDLED">true to enable PDL edit distance</param>
/// <param name="enableDensity">true to enable density distance</param>
/// <returns>Grade for dfa2</returns>
public static Pair <int, IEnumerable <DFAFeedback> > GetGrade(
Automaton <BDD> dfaGoal, Automaton <BDD> dfaAttempt, HashSet <char> al,
CharSetSolver solver, long timeout,
int maxGrade, FeedbackLevel level,
bool enableDFAED, bool enablePDLED, bool enableDensity)
{
PDLEnumerator pdlEnumerator = new PDLEnumerator();
var feedList = new List <DFAFeedback>();
DFAFeedback defaultFeedback = new StringFeedback(level, StringFeedbackType.Wrong, al, solver);
#region Accessory and initial vars
//Compute minimized version of DFAs
var dfaGoalMin = dfaGoal.Determinize(solver).Minimize(solver);
var dfaAttemptMin = dfaAttempt.Determinize(solver).Minimize(solver);
//Initialize distances at high values in case they are not used
// they only produce positive grade if between 0 and 1
double pdlEditDistanceScaled = 2;
double densityRatio = 2;
double dfaED = 2;
#endregion
#region deductions on the grade based on the size of the dfa
//Deduction if DFA is smaller than it should be: used only for PDL ed and for density
var smallerDFADeduction = 0.2 * Math.Sqrt(
Math.Max(0.0, dfaGoalMin.StateCount - dfaAttemptMin.StateCount) /
((double)dfaGoalMin.StateCount));
#endregion
#region check whether the attempt is equivalent to the solution
if (dfaGoal.IsEquivalentWith(dfaAttempt, solver))
{
Console.WriteLine("Correct");
feedList.Add(new StringFeedback(level, StringFeedbackType.Correct, al, solver));
return(new Pair <int, IEnumerable <DFAFeedback> >(maxGrade, feedList));
}
#endregion
#region metrics computation
Stopwatch swPDLed = new Stopwatch();
swPDLed.Start();
#region PDL edit distance
Transformation feedbackTransformation = null;
if (enablePDLED)
{
var trpair = PDLEditDistance.GetMinimalFormulaEditDistanceTransformation(dfaGoalMin, dfaAttemptMin, al, solver, timeout, pdlEnumerator);
if (trpair != null)
{
var transformationGrade = trpair.First;
feedbackTransformation = trpair.Second;
var scaling = 1.0;
pdlEditDistanceScaled = transformationGrade.totalCost / (transformationGrade.minSizeForTreeA * scaling) + smallerDFADeduction;
}
}
#endregion
swPDLed.Stop();
Stopwatch swDensity = new Stopwatch();
swDensity.Start();
#region density distance
if (enableDensity)
{
densityRatio = DFADensity.GetDFADifferenceRatio(dfaGoalMin, dfaAttemptMin, al, solver);
densityRatio += smallerDFADeduction;
}
#endregion
swDensity.Stop();
Stopwatch swDFAed = new Stopwatch();
swDFAed.Start();
#region DFA edit distance
DFAEditScript dfaEditScript = null;
if (enableDFAED)
{
//limit the depth of the DFA edit distance search
var maxMoves = Math.Max(1, 6 - (int)Math.Sqrt(dfaAttempt.MoveCount + dfaAttempt.StateCount));
dfaEditScript = DFAEditDistance.GetDFAOptimalEdit(dfaGoal, dfaAttempt, al, solver, timeout, new StringBuilder());
if (dfaEditScript != null)
{
dfaED = ((double)(dfaEditScript.GetCost())) / ((double)((dfaGoalMin.StateCount + 1) * al.Count));
}
}
#endregion
swDFAed.Stop();
#endregion
#region metrics scaling
var scalingSquarePDLED = 1.005;
var scalingSquareDensity = 1; var multv2 = 0.5;
var scalingSquareDFAED = 1.03;
var scaledPdlED = (0.9 * (scalingSquarePDLED + pdlEditDistanceScaled) * (scalingSquarePDLED + pdlEditDistanceScaled)) - scalingSquarePDLED * scalingSquarePDLED;
var scaledDensityRatio = (scalingSquareDensity + (multv2 * densityRatio)) * (scalingSquareDensity + (multv2 * densityRatio)) - scalingSquareDensity * scalingSquareDensity;
var scaledDfaED = (scalingSquareDFAED + dfaED) * (scalingSquareDFAED + dfaED) - scalingSquareDFAED * scalingSquareDFAED;
//Select dominating Feedback based on grade
double unscaledGrade = Math.Min(Math.Min(scaledPdlED, scaledDensityRatio), scaledDfaED);
var pdledwins = scaledPdlED <= Math.Min(scaledDensityRatio, scaledDfaED);
var dfaedwins = scaledDfaED <= Math.Min(scaledDensityRatio, scaledPdlED);
var densitywins = scaledDensityRatio <= Math.Min(scaledDfaED, scaledPdlED);
#endregion
#region Feedback Selection
if (pdledwins && feedbackTransformation != null && feedbackTransformation.pdlB.GetFormulaSize() < 10)
{
feedList.Add(new PDLEDFeedback(level, al, feedbackTransformation, scaledPdlED, solver));
}
if ((dfaedwins || feedList.Count == 0) && dfaEditScript != null && !dfaEditScript.IsComplex())
{
feedList = new List <DFAFeedback>();
feedList.Add(new DFAEDFeedback(dfaGoal, dfaAttempt, level, al, dfaEditScript, scaledDfaED, solver));
}
if (densitywins || feedList.Count == 0)
{
feedList = new List <DFAFeedback>();
feedList.Add(new DensityFeedback(level, al, dfaGoal, dfaAttempt, scaledDensityRatio, solver));
}
if (feedList.Count == 0)
{
Console.WriteLine("Why no feedback!!");
feedList.Add(defaultFeedback);
}
#endregion
#region normalize grade
var scaledGrade = maxGrade - (int)Math.Round(unscaledGrade * (double)(maxGrade));
//If rounding yields maxgrade deduct 1 point by default
if (scaledGrade == maxGrade)
{
scaledGrade = maxGrade - 1;
}
//Remove possible deduction
scaledGrade = scaledGrade < 0 ? 0 : scaledGrade;
return(new Pair <int, IEnumerable <DFAFeedback> >(scaledGrade, feedList));
#endregion
}
/// <summary>
/// Finds min edit distance script between DFAs if operation
/// takes less than timeout ms
/// </summary>
/// <param name="dfa1"></param>
/// <param name="dfa2"></param>
/// <param name="al"></param>
/// <param name="solver"></param>
/// <param name="timeout"></param>
/// <param name="sb"></param>
/// <returns></returns>
public static DFAEditScript GetDFAOptimalEdit( // copy
Automaton<BDD> dfa1, Automaton<BDD> dfa2,
HashSet<char> al, CharSetSolver solver, long timeout,
StringBuilder sb)
{
//Contract.Assert(dfa1.IsDeterministic);
//Contract.Assert(dfa2.IsDeterministic);
DFAEditScript editScript = new DFAEditScript();
#region Add states to dfa2 to make it at least as dfa1
BDD fullAlphabetCondition = BDDOf(al, solver);
//Normalize the DFA giving only names from 0 to |States|-1
var normDfaPair = DFAUtilities.normalizeDFA(dfa2);
var dfa2augmented = normDfaPair.First;
//solver.SaveAsDot(dfa2augmented, "aaaa");
var stateNamesMapping = normDfaPair.Second;
//Add states to make dfa2 have the |dfa2.States|>= |dfa1.States|
var newMoves = new List<Move<BDD>>(dfa2augmented.GetMoves());
for (int i = 1; i <= dfa1.StateCount - dfa2augmented.StateCount; i++)
{
int newStateName = dfa2augmented.MaxState + i;
//Pick the next available name to be added
stateNamesMapping[newStateName] = dfa2.MaxState + i;
//save the operation in the script
editScript.script.Insert(0, new DFAAddState(dfa2.MaxState + i));
newMoves.Add(new Move<BDD>(newStateName, newStateName, fullAlphabetCondition));
newStateName++;
}
//Create the new DFA with the added states
dfa2augmented = Automaton<BDD>.Create(dfa2augmented.InitialState, dfa2augmented.GetFinalStates().ToList(), newMoves);
#endregion
int maxScore = (dfa1.StateCount + dfa2augmented.StateCount) * (al.Count + 1);
int oldScirptSize = editScript.script.Count;
//Start with the internal script equals to null, at the end bestScript.Script will contain the best script
DFAEditScript bestScript = new DFAEditScript();
bestScript.script = null;
Stopwatch sw = new Stopwatch();
sw.Start();
// Iteratively check if there exists an edit of a given depth
for (int depth = 1; true; depth++)
{
var editList = new List<DFAEdit>();
if (GetDFAEditScriptTimeout(
dfa1, dfa2augmented, al, solver,
new List<long>(), editScript.script,
depth, timeout, sw, DFAUtilities.MyHillTestGeneration(al, dfa1, solver),
DFADensity.GetDFADensity(dfa1, al, solver),
editScript.GetCost(), bestScript, stateNamesMapping))
{
// if hits timeout break and return null
break;
}
if (bestScript.script != null)
{
bestScript.script.Reverse();
sw.Stop();
return bestScript;
}
}
sw.Stop();
return null;
}
// looks for an edit at depth "depth"
// returns false and null in bestScript if no edit is found at depth "depth"
// returns false and not null in bestScript if found
// returns true if timeout
internal static bool GetDFAEditScriptTimeout(
Automaton<BDD> dfa1, Automaton<BDD> dfa2,
HashSet<char> al, CharSetSolver solver,
List<long> editScriptHash, List<DFAEdit> editList,
int depth, long timeout, Stopwatch sw,
Pair<IEnumerable<string>, IEnumerable<string>> tests,
double dfa1density,
int totalCost,
DFAEditScript bestScript, Dictionary<int, int> stateNamesMapping)
{
// check timer
if (sw.ElapsedMilliseconds > timeout)
return true;
//Compute worst case distance, call finalScript with this value?
int dist = (dfa1.StateCount + dfa2.StateCount) * (al.Count + 1);
//Stop if no more moves left
if (depth == 0)
{
if (DFAUtilities.ApproximateMNEquivalent(tests, dfa1density, dfa2, al, solver) && dfa2.IsEquivalentWith(dfa1, solver))
//check if totalCost < finalScript cost and replace if needed
if (bestScript.script == null || totalCost < bestScript.GetCost())
bestScript.script = ObjectCopier.Clone<List<DFAEdit>>(editList);
return false;
}
DFAEdit edit = null;
#region Flip one move target state
foreach (var move in dfa2.GetMoves())
{
//Creaty copy of the moves without current move
var movesWithoutCurrMove = dfa2.GetMoves().ToList();
movesWithoutCurrMove.Remove(move);
//Redirect every ch belonging to move condition
foreach (var c in solver.GenerateAllCharacters(move.Label, false))
{
long hash = IntegerUtil.PairToInt(move.SourceState, c - 97) + dfa2.StateCount;
if (CanAdd(hash, editScriptHash))
{
editScriptHash.Insert(0, hash);
//Local copy of moves
var newMoves = movesWithoutCurrMove.ToList();
var newMoveCondition = solver.MkCharConstraint(false, c);
#region Remove ch from current move
var andCond = solver.MkAnd(move.Label, solver.MkNot(newMoveCondition));
//add back move without ch iff satisfiable
if (solver.IsSatisfiable(andCond))
newMoves.Add(new Move<BDD>(move.SourceState, move.TargetState, andCond));
#endregion
#region Redirect c to a different state
foreach (var state in dfa2.States)
if (state != move.TargetState)
{
var newMovesComplete = newMoves.ToList();
newMovesComplete.Add(new Move<BDD>(move.SourceState, state, newMoveCondition));
var dfa2new = Automaton<BDD>.Create(dfa2.InitialState, dfa2.GetFinalStates(), newMovesComplete);
edit = new DFAEditMove(stateNamesMapping[move.SourceState], stateNamesMapping[state], c);
editList.Insert(0, edit);
if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
return true;
editList.RemoveAt(0);
}
#endregion
editScriptHash.RemoveAt(0);
}
}
}
#endregion
#region Flip one state from fin to non fin
foreach (var state in dfa2.States)
{
if (CanAdd(state, editScriptHash))
{
//flip its final non final status
editScriptHash.Insert(0, state);
var newFinalStates = new HashSet<int>(dfa2.GetFinalStates());
Automaton<BDD> dfa2new = null;
if (dfa2.GetFinalStates().Contains(state))
{
edit = new DFAEditState(stateNamesMapping[state], false);
editList.Insert(0, edit);
newFinalStates.Remove(state);
dfa2new = Automaton<BDD>.Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
}
else
{
edit = new DFAEditState(stateNamesMapping[state], true);
editList.Insert(0, edit);
newFinalStates.Add(state);
dfa2new = Automaton<BDD>.Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
}
if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
return true;
editScriptHash.RemoveAt(0);
editList.RemoveAt(0);
}
}
#endregion
return false;
}
/// <summary>
/// Finds min edit distance script between DFAs if operation
/// takes less than timeout ms
/// </summary>
/// <param name="dfa1"></param>
/// <param name="dfa2"></param>
/// <param name="al"></param>
/// <param name="solver"></param>
/// <param name="timeout"></param>
/// <param name="sb"></param>
/// <returns></returns>
public static DFAEditScript GetDFAOptimalEdit( // copy
Automaton <BDD> dfa1, Automaton <BDD> dfa2,
HashSet <char> al, CharSetSolver solver, long timeout,
StringBuilder sb)
{
//Contract.Assert(dfa1.IsDeterministic);
//Contract.Assert(dfa2.IsDeterministic);
DFAEditScript editScript = new DFAEditScript();
#region Add states to dfa2 to make it at least as dfa1
BDD fullAlphabetCondition = BDDOf(al, solver);
//Normalize the DFA giving only names from 0 to |States|-1
var normDfaPair = DFAUtilities.normalizeDFA(dfa2);
var dfa2augmented = normDfaPair.First;
//solver.SaveAsDot(dfa2augmented, "aaaa");
var stateNamesMapping = normDfaPair.Second;
//Add states to make dfa2 have the |dfa2.States|>= |dfa1.States|
var newMoves = new List <Move <BDD> >(dfa2augmented.GetMoves());
for (int i = 1; i <= dfa1.StateCount - dfa2augmented.StateCount; i++)
{
int newStateName = dfa2augmented.MaxState + i;
//Pick the next available name to be added
stateNamesMapping[newStateName] = dfa2.MaxState + i;
//save the operation in the script
editScript.script.Insert(0, new DFAAddState(dfa2.MaxState + i));
newMoves.Add(new Move <BDD>(newStateName, newStateName, fullAlphabetCondition));
newStateName++;
}
//Create the new DFA with the added states
dfa2augmented = Automaton <BDD> .Create(dfa2augmented.InitialState, dfa2augmented.GetFinalStates().ToList(), newMoves);
#endregion
int maxScore = (dfa1.StateCount + dfa2augmented.StateCount) * (al.Count + 1);
int oldScirptSize = editScript.script.Count;
//Start with the internal script equals to null, at the end bestScript.Script will contain the best script
DFAEditScript bestScript = new DFAEditScript();
bestScript.script = null;
Stopwatch sw = new Stopwatch();
sw.Start();
// Iteratively check if there exists an edit of a given depth
for (int depth = 1; true; depth++)
{
var editList = new List <DFAEdit>();
if (GetDFAEditScriptTimeout(
dfa1, dfa2augmented, al, solver,
new List <long>(), editScript.script,
depth, timeout, sw, DFAUtilities.MyHillTestGeneration(al, dfa1, solver),
DFADensity.GetDFADensity(dfa1, al, solver),
editScript.GetCost(), bestScript, stateNamesMapping))
{
// if hits timeout break and return null
break;
}
if (bestScript.script != null)
{
bestScript.script.Reverse();
sw.Stop();
return(bestScript);
}
}
sw.Stop();
return(null);
}
// looks for an edit at depth "depth"
// returns false and null in bestScript if no edit is found at depth "depth"
// returns false and not null in bestScript if found
// returns true if timeout
internal static bool GetDFAEditScriptTimeout(
Automaton <BDD> dfa1, Automaton <BDD> dfa2,
HashSet <char> al, CharSetSolver solver,
List <long> editScriptHash, List <DFAEdit> editList,
int depth, long timeout, Stopwatch sw,
Pair <IEnumerable <string>, IEnumerable <string> > tests,
double dfa1density,
int totalCost,
DFAEditScript bestScript, Dictionary <int, int> stateNamesMapping)
{
// check timer
if (sw.ElapsedMilliseconds > timeout)
{
return(true);
}
//Compute worst case distance, call finalScript with this value?
int dist = (dfa1.StateCount + dfa2.StateCount) * (al.Count + 1);
//Stop if no more moves left
if (depth == 0)
{
if (DFAUtilities.ApproximateMNEquivalent(tests, dfa1density, dfa2, al, solver) && dfa2.IsEquivalentWith(dfa1, solver))
{
//check if totalCost < finalScript cost and replace if needed
if (bestScript.script == null || totalCost < bestScript.GetCost())
{
bestScript.script = ObjectCopier.Clone <List <DFAEdit> >(editList);
}
}
return(false);
}
DFAEdit edit = null;
#region Flip one move target state
foreach (var move in dfa2.GetMoves())
{
//Creaty copy of the moves without current move
var movesWithoutCurrMove = dfa2.GetMoves().ToList();
movesWithoutCurrMove.Remove(move);
//Redirect every ch belonging to move condition
foreach (var c in solver.GenerateAllCharacters(move.Label, false))
{
long hash = IntegerUtil.PairToInt(move.SourceState, c - 97) + dfa2.StateCount;
if (CanAdd(hash, editScriptHash))
{
editScriptHash.Insert(0, hash);
//Local copy of moves
var newMoves = movesWithoutCurrMove.ToList();
var newMoveCondition = solver.MkCharConstraint(false, c);
#region Remove ch from current move
var andCond = solver.MkAnd(move.Label, solver.MkNot(newMoveCondition));
//add back move without ch iff satisfiable
if (solver.IsSatisfiable(andCond))
{
newMoves.Add(new Move <BDD>(move.SourceState, move.TargetState, andCond));
}
#endregion
#region Redirect c to a different state
foreach (var state in dfa2.States)
{
if (state != move.TargetState)
{
var newMovesComplete = newMoves.ToList();
newMovesComplete.Add(new Move <BDD>(move.SourceState, state, newMoveCondition));
var dfa2new = Automaton <BDD> .Create(dfa2.InitialState, dfa2.GetFinalStates(), newMovesComplete);
edit = new DFAEditMove(stateNamesMapping[move.SourceState], stateNamesMapping[state], c);
editList.Insert(0, edit);
if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
{
return(true);
}
editList.RemoveAt(0);
}
}
#endregion
editScriptHash.RemoveAt(0);
}
}
}
#endregion
#region Flip one state from fin to non fin
foreach (var state in dfa2.States)
{
if (CanAdd(state, editScriptHash))
{
//flip its final non final status
editScriptHash.Insert(0, state);
var newFinalStates = new HashSet <int>(dfa2.GetFinalStates());
Automaton <BDD> dfa2new = null;
if (dfa2.GetFinalStates().Contains(state))
{
edit = new DFAEditState(stateNamesMapping[state], false);
editList.Insert(0, edit);
newFinalStates.Remove(state);
dfa2new = Automaton <BDD> .Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
}
else
{
edit = new DFAEditState(stateNamesMapping[state], true);
editList.Insert(0, edit);
newFinalStates.Add(state);
dfa2new = Automaton <BDD> .Create(dfa2.InitialState, newFinalStates, dfa2.GetMoves());
}
if (GetDFAEditScriptTimeout(dfa1, dfa2new, al, solver, editScriptHash, editList, depth - 1, timeout, sw, tests, dfa1density, totalCost + edit.GetCost(), bestScript, stateNamesMapping))
{
return(true);
}
editScriptHash.RemoveAt(0);
editList.RemoveAt(0);
}
}
#endregion
return(false);
}
