/// <summary>
/// Computes the ratio of the symmetric difference to the size of dfa1 enumerating paths up to length n (uses the complement if density is high)
/// </summary>
/// <returns>size of ((dfa2-dfa1)+(dfa1-dfa2))/dfa1</returns>
public static double GetDFADifferenceRatio(Automaton <BDD> dfa1, Automaton <BDD> dfa2, HashSet <char> al, CharSetSolver solver)
{
var solutionDensity = DFADensity.GetDFADensity(dfa1, al, solver);
//Symmetric difference
var dfadiff1 = dfa1.Minus(dfa2, solver);
var dfadiff2 = dfa2.Minus(dfa1, solver);
var dfatrue = Automaton <BDD> .Create(0, new int[] { 0 }, new Move <BDD>[] { new Move <BDD>(0, 0, solver.True) });
var dfadiff = dfatrue.Minus(dfatrue.Minus(dfadiff1, solver).Intersect(dfatrue.Minus(dfadiff2, solver), solver), solver).Determinize(solver).Minimize(solver);
//Use smallest of |dfa1| and complement of |dfa1| for cardinality base
return(GetDFARatio(dfa1.Determinize(solver).Minimize(solver), dfadiff, al, solver, solutionDensity > 0.5));
}
public static bool canCollapseStates(Automaton <BDD> nfa, int state1, int state2, CharSetSolver solver, Pair <IEnumerable <string>, IEnumerable <string> > tests, HashSet <char> al)
{
var density = DFADensity.GetDFADensity(nfa, al, solver);
// collapses state2 to state1
List <Move <BDD> > newMoves = new List <Move <BDD> >();
foreach (var move in nfa.GetMoves())
{
var newSource = move.SourceState;
var newTarget = move.TargetState;
if (newSource == state2)
{
newSource = state1;
}
if (newTarget == state2)
{
newTarget = state1;
}
newMoves.Add(new Move <BDD>(newSource, newTarget, move.Label));
}
// replace state2 with state1 if initial state
// no need to remove state2 from final state list, as it is unreachable
int newInitialState = nfa.InitialState;
if (nfa.InitialState == state2)
{
newInitialState = state1;
}
//makes new Nfa and returns collapse state edit if are equiv
var newNfa = Automaton <BDD> .Create(newInitialState, nfa.GetFinalStates(), newMoves);
if (DFAUtilities.ApproximateMNEquivalent(tests, density, newNfa, al, solver))
{
return(nfa.IsEquivalentWith(newNfa, solver));
}
return(false);
}
/// <summary>
/// create new instance of NFAEdit distance and assigns a number to each character
/// </summary>
/// <param name="nfa1"></param>
/// <param name="nfa2"></param>
/// <param name="al"></param>
/// <param name="solver"></param>
/// <param name="timeout"></param>
public NFAEditDistanceProvider(Automaton <BDD> nfa1,
HashSet <char> al, CharSetSolver solver, long timeout)
{
this.nfa1 = nfa1;
this.al = al;
this.solver = solver;
this.timeout = timeout;
this.alphabetMap = new Dictionary <char, int>();
int index = 0;
foreach (var c in al)
{
this.alphabetMap[c] = index;
index++;
}
this.sw = new Stopwatch();
this.tests = NFAUtilities.MyHillTestGeneration(al, nfa1.Determinize(solver), solver);
var dfa1 = nfa1.RemoveEpsilons(solver.MkOr).Determinize(solver).Minimize(solver);
this.nfa1density = DFADensity.GetDFADensity(dfa1, al, solver);
}
/// <summary>
/// Computes the grade for attempt using all the possible metrics
/// </summary>
/// <param name="solutionNFA">correct nfa</param>
/// <param name="attemptNFA">nfa to be graded</param>
/// <param name="alpahbet">input alphabet</param>
/// <param name="solver">SMT solver for char set</param>
/// <param name="timeout">timeout for the PDL enumeration (suggested > 1000)</param>
/// <param name="maxGrade">Max grade for the homework</param>
/// <param name="level">Feedback level</param>
/// <returns>Grade for nfa2</returns>
public static Pair <int, IEnumerable <NFAFeedback> > GetGrade(
Automaton <BDD> solutionNFA, Automaton <BDD> attemptNFA, HashSet <char> alphabet,
CharSetSolver solver, long timeout, int maxGrade, FeedbackLevel level)
{
var feedbacks = new List <NFAFeedback>();
int deadStateDeduction = 0;
int tooBigDeduction = 0;
int incorrectDeduction = 0;
// Remove at most a percentage of max grade when NFA is big
double maxDeductionForTooBig = ((double)maxGrade * 0.3);
double maxDeductionForDeadStates = ((double)maxGrade * 0.1);
double solutionStateCount = solutionNFA.StateCount;
double attemptStateCount = attemptNFA.StateCount;
double solutionTransCount = solutionNFA.MoveCount;
double attemptTransCount = attemptNFA.MoveCount;
NFAEditDistanceProvider nfaedp = new NFAEditDistanceProvider(solutionNFA, alphabet, solver, timeout);
//Check if using epsilon and nondeterminism
if (solutionNFA.IsEpsilonFree)
{
solutionNFA.CheckDeterminism(solver);
}
if (attemptNFA.IsEpsilonFree)
{
attemptNFA.CheckDeterminism(solver);
}
bool shouldUseEpsilon = !solutionNFA.IsEpsilonFree && attemptNFA.IsEpsilonFree;
bool shouldUseNonDet = !shouldUseEpsilon && !solutionNFA.isDeterministic && attemptNFA.isDeterministic;
//Check if solution has dead states and remove if it does
var statesBeforeDeadStatesElimination = attemptNFA.StateCount;
attemptNFA.EliminateDeadStates();
var solutionHasDeadStates = attemptNFA.StateCount < statesBeforeDeadStatesElimination;
//Start checking equiv
bool areEquivalent = solutionNFA.IsEquivalentWith(attemptNFA, solver);
if (areEquivalent)
{
// prompt nfa is correct
feedbacks.Insert(0, new NFAStringFeedback(level, alphabet, solver, "Your NFA accepts the CORRECT language."));
#region Check number of states and decrease grade if too big
int stateDiff = (int)(attemptStateCount - solutionStateCount);
int transDiff = (int)(attemptTransCount - solutionTransCount);
//If is not minimal apply deduction and compute edit
if (stateDiff > 0 || transDiff > 0)
{
#region Try to collapse for feedback
// Try to find a way to collaps states or remove states and transitions to make the NFA smaller
NFAEditScript collapseScript = null;
var edit = nfaedp.NFACollapseSearch(attemptNFA);
if (edit != null)
{
collapseScript = new NFAEditScript();
collapseScript.script.Insert(0, edit);
}
feedbacks.Add(new NFANotMinimalFeedback(level, alphabet, stateDiff, transDiff, collapseScript, solver));
#endregion
#region Compute tooBigDeduction
if (stateDiff > 0)
{
// ((att/sol)^2)-1
var stateRatio = attemptStateCount / solutionStateCount;
var stateRatioSqM1 = Math.Pow(stateRatio, 2) - 1;
var sclaedStateRatio = stateRatioSqM1 * maxDeductionForTooBig / 2;
tooBigDeduction = (int)Math.Round(Math.Min(sclaedStateRatio, maxDeductionForTooBig));
}
else
{
if (transDiff > 0)
{
// ((att/sol)^2)-1
var transRatio = attemptTransCount / solutionTransCount;
var transRatioSqM1 = Math.Pow(transRatio, 2) - 1;
var sclaedTransRatio = transRatioSqM1 * maxDeductionForTooBig / 2;
tooBigDeduction = (int)Math.Round(Math.Min(sclaedTransRatio, maxDeductionForTooBig));
}
}
//Make sure deduction is positive
tooBigDeduction = Math.Max(tooBigDeduction, 0);
#endregion
}
#endregion
}
else
{
// prompt nfa is incorrect
feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "Your NFA does NOT accept the correct langauge."));
//inequivalent, try using grading metrics and based on winning metric give feedback
int remainingGrade = maxGrade - tooBigDeduction;
#region metric computation
//compute deterministic versions
var solutionNFAdet = solutionNFA.RemoveEpsilons(solver.MkOr).Determinize(solver).MakeTotal(solver).Minimize(solver);
var attemptNFAdet = attemptNFA.RemoveEpsilons(solver.MkOr).Determinize(solver).MakeTotal(solver).Minimize(solver);
solutionNFAdet.EliminateDeadStates();
attemptNFAdet.EliminateDeadStates();
//compute density
double densityRatio = DFADensity.GetDFADifferenceRatio(solutionNFAdet, attemptNFAdet, alphabet, solver);
//compute edit distance
double nfaED = 2;
var editScript = nfaedp.GetNFAOptimalEdit(attemptNFA);
if (editScript != null)
{
nfaED = ((double)(editScript.GetCost())) / ((double)((solutionNFA.StateCount + 1) * alphabet.Count));
}
#endregion
#region metrics scaling
var scalingSquareDensity = 1; var multv2 = 0.5;
var scalingSquareDFAED = 1.03;
var scaledDensityRatio = (scalingSquareDensity + (multv2 * densityRatio)) * (scalingSquareDensity + (multv2 * densityRatio)) - scalingSquareDensity * scalingSquareDensity;
var scaledNfaED = (scalingSquareDFAED + nfaED) * (scalingSquareDFAED + nfaED) - scalingSquareDFAED * scalingSquareDFAED;
//If the edit script was not computed make sure it loses.
if (editScript == null)
{
scaledNfaED = Double.MaxValue;
}
//Select dominating Feedback based on grade
double unscaledGrade = Math.Min(scaledDensityRatio, scaledNfaED);
var dfaedwins = scaledNfaED <= scaledDensityRatio;
var densitywins = scaledDensityRatio <= scaledNfaED;
incorrectDeduction = (int)Math.Round(unscaledGrade * (double)(maxGrade));
#endregion
//If edit distance search works, provides feedback based upon result
//Otherwise, gives counterexample feedback
if (level != FeedbackLevel.Minimal)
{
if (dfaedwins)
{
feedbacks.Add(new NFAEDFeedback(solutionNFA, attemptNFA, level, alphabet, editScript, solver));
}
else
{
feedbacks.Add(new NFACounterexampleFeedback(level, alphabet, solutionNFAdet, attemptNFAdet, solver));
}
}
}
// Feedback related to nondeterminism and epislon
if (shouldUseEpsilon)
{
feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "You should try using epsilon transitions."));
}
if (shouldUseNonDet)
{
feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "You should try using nondeterminism."));
}
// Deduct points and prompt feedback is solution has dead states
if (solutionHasDeadStates)
{
deadStateDeduction = (int)maxDeductionForDeadStates;
feedbacks.Add(new NFAStringFeedback(level, alphabet, solver, "Your NFA has some dead states."));
}
//Grade computation
//changed to be binary, because we did not tell them of the deductions
//int grade = Math.Max(maxGrade - deadStateDeduction - tooBigDeduction - incorrectDeduction, 0);
//int grade = areEquivalent ? maxGrade : 0;
// This requires further testing, but appears to grade ok, 0 points for empty automaton and full points for something in between
int grade = Math.Max(maxGrade - incorrectDeduction, 0);
return(new Pair <int, IEnumerable <NFAFeedback> >(grade, feedbacks));
}
/// <summary>
/// Computes the grade for attempt using all the possible metrics
/// </summary>
/// <param name="dfaGoal">minimal correct dfa</param>
/// <param name="dfaAttempt">dfa to be graded</param>
/// <param name="al">input alphabet</param>
/// <param name="solver">SMT solver for char set</param>
/// <param name="timeout">timeout for the PDL enumeration (suggested > 1000)</param>
/// <param name="maxGrade">Max grade for the homework</param>
/// <param name="enableDFAED">true to enable DFA edit distance</param>
/// <param name="enablePDLED">true to enable PDL edit distance</param>
/// <param name="enableDensity">true to enable density distance</param>
/// <returns>Grade for dfa2</returns>
public static Pair <int, IEnumerable <DFAFeedback> > GetGrade(
Automaton <BDD> dfaGoal, Automaton <BDD> dfaAttempt, HashSet <char> al,
CharSetSolver solver, long timeout,
int maxGrade, FeedbackLevel level,
bool enableDFAED, bool enablePDLED, bool enableDensity)
{
PDLEnumerator pdlEnumerator = new PDLEnumerator();
var feedList = new List <DFAFeedback>();
DFAFeedback defaultFeedback = new StringFeedback(level, StringFeedbackType.Wrong, al, solver);
#region Accessory and initial vars
//Compute minimized version of DFAs
var dfaGoalMin = dfaGoal.Determinize(solver).Minimize(solver);
var dfaAttemptMin = dfaAttempt.Determinize(solver).Minimize(solver);
//Initialize distances at high values in case they are not used
// they only produce positive grade if between 0 and 1
double pdlEditDistanceScaled = 2;
double densityRatio = 2;
double dfaED = 2;
#endregion
#region deductions on the grade based on the size of the dfa
//Deduction if DFA is smaller than it should be: used only for PDL ed and for density
var smallerDFADeduction = 0.2 * Math.Sqrt(
Math.Max(0.0, dfaGoalMin.StateCount - dfaAttemptMin.StateCount) /
((double)dfaGoalMin.StateCount));
#endregion
#region check whether the attempt is equivalent to the solution
if (dfaGoal.IsEquivalentWith(dfaAttempt, solver))
{
Console.WriteLine("Correct");
feedList.Add(new StringFeedback(level, StringFeedbackType.Correct, al, solver));
return(new Pair <int, IEnumerable <DFAFeedback> >(maxGrade, feedList));
}
#endregion
#region metrics computation
Stopwatch swPDLed = new Stopwatch();
swPDLed.Start();
#region PDL edit distance
Transformation feedbackTransformation = null;
if (enablePDLED)
{
var trpair = PDLEditDistance.GetMinimalFormulaEditDistanceTransformation(dfaGoalMin, dfaAttemptMin, al, solver, timeout, pdlEnumerator);
if (trpair != null)
{
var transformationGrade = trpair.First;
feedbackTransformation = trpair.Second;
var scaling = 1.0;
pdlEditDistanceScaled = transformationGrade.totalCost / (transformationGrade.minSizeForTreeA * scaling) + smallerDFADeduction;
}
}
#endregion
swPDLed.Stop();
Stopwatch swDensity = new Stopwatch();
swDensity.Start();
#region density distance
if (enableDensity)
{
densityRatio = DFADensity.GetDFADifferenceRatio(dfaGoalMin, dfaAttemptMin, al, solver);
densityRatio += smallerDFADeduction;
}
#endregion
swDensity.Stop();
Stopwatch swDFAed = new Stopwatch();
swDFAed.Start();
#region DFA edit distance
DFAEditScript dfaEditScript = null;
if (enableDFAED)
{
//limit the depth of the DFA edit distance search
var maxMoves = Math.Max(1, 6 - (int)Math.Sqrt(dfaAttempt.MoveCount + dfaAttempt.StateCount));
dfaEditScript = DFAEditDistance.GetDFAOptimalEdit(dfaGoal, dfaAttempt, al, solver, timeout, new StringBuilder());
if (dfaEditScript != null)
{
dfaED = ((double)(dfaEditScript.GetCost())) / ((double)((dfaGoalMin.StateCount + 1) * al.Count));
}
}
#endregion
swDFAed.Stop();
#endregion
#region metrics scaling
var scalingSquarePDLED = 1.005;
var scalingSquareDensity = 1; var multv2 = 0.5;
var scalingSquareDFAED = 1.03;
var scaledPdlED = (0.9 * (scalingSquarePDLED + pdlEditDistanceScaled) * (scalingSquarePDLED + pdlEditDistanceScaled)) - scalingSquarePDLED * scalingSquarePDLED;
var scaledDensityRatio = (scalingSquareDensity + (multv2 * densityRatio)) * (scalingSquareDensity + (multv2 * densityRatio)) - scalingSquareDensity * scalingSquareDensity;
var scaledDfaED = (scalingSquareDFAED + dfaED) * (scalingSquareDFAED + dfaED) - scalingSquareDFAED * scalingSquareDFAED;
//Select dominating Feedback based on grade
double unscaledGrade = Math.Min(Math.Min(scaledPdlED, scaledDensityRatio), scaledDfaED);
var pdledwins = scaledPdlED <= Math.Min(scaledDensityRatio, scaledDfaED);
var dfaedwins = scaledDfaED <= Math.Min(scaledDensityRatio, scaledPdlED);
var densitywins = scaledDensityRatio <= Math.Min(scaledDfaED, scaledPdlED);
#endregion
#region Feedback Selection
if (pdledwins && feedbackTransformation != null && feedbackTransformation.pdlB.GetFormulaSize() < 10)
{
feedList.Add(new PDLEDFeedback(level, al, feedbackTransformation, scaledPdlED, solver));
}
if ((dfaedwins || feedList.Count == 0) && dfaEditScript != null && !dfaEditScript.IsComplex())
{
feedList = new List <DFAFeedback>();
feedList.Add(new DFAEDFeedback(dfaGoal, dfaAttempt, level, al, dfaEditScript, scaledDfaED, solver));
}
if (densitywins || feedList.Count == 0)
{
feedList = new List <DFAFeedback>();
feedList.Add(new DensityFeedback(level, al, dfaGoal, dfaAttempt, scaledDensityRatio, solver));
}
if (feedList.Count == 0)
{
Console.WriteLine("Why no feedback!!");
feedList.Add(defaultFeedback);
}
#endregion
#region normalize grade
var scaledGrade = maxGrade - (int)Math.Round(unscaledGrade * (double)(maxGrade));
//If rounding yields maxgrade deduct 1 point by default
if (scaledGrade == maxGrade)
{
scaledGrade = maxGrade - 1;
}
//Remove possible deduction
scaledGrade = scaledGrade < 0 ? 0 : scaledGrade;
return(new Pair <int, IEnumerable <DFAFeedback> >(scaledGrade, feedList));
#endregion
}
/// <summary>
/// Finds min edit distance script between DFAs if operation
/// takes less than timeout ms
/// </summary>
/// <param name="dfa1"></param>
/// <param name="dfa2"></param>
/// <param name="al"></param>
/// <param name="solver"></param>
/// <param name="timeout"></param>
/// <param name="sb"></param>
/// <returns></returns>
public static DFAEditScript GetDFAOptimalEdit( // copy
Automaton <BDD> dfa1, Automaton <BDD> dfa2,
HashSet <char> al, CharSetSolver solver, long timeout,
StringBuilder sb)
{
//Contract.Assert(dfa1.IsDeterministic);
//Contract.Assert(dfa2.IsDeterministic);
DFAEditScript editScript = new DFAEditScript();
#region Add states to dfa2 to make it at least as dfa1
BDD fullAlphabetCondition = BDDOf(al, solver);
//Normalize the DFA giving only names from 0 to |States|-1
var normDfaPair = DFAUtilities.normalizeDFA(dfa2);
var dfa2augmented = normDfaPair.First;
//solver.SaveAsDot(dfa2augmented, "aaaa");
var stateNamesMapping = normDfaPair.Second;
//Add states to make dfa2 have the |dfa2.States|>= |dfa1.States|
var newMoves = new List <Move <BDD> >(dfa2augmented.GetMoves());
for (int i = 1; i <= dfa1.StateCount - dfa2augmented.StateCount; i++)
{
int newStateName = dfa2augmented.MaxState + i;
//Pick the next available name to be added
stateNamesMapping[newStateName] = dfa2.MaxState + i;
//save the operation in the script
editScript.script.Insert(0, new DFAAddState(dfa2.MaxState + i));
newMoves.Add(new Move <BDD>(newStateName, newStateName, fullAlphabetCondition));
newStateName++;
}
//Create the new DFA with the added states
dfa2augmented = Automaton <BDD> .Create(dfa2augmented.InitialState, dfa2augmented.GetFinalStates().ToList(), newMoves);
#endregion
int maxScore = (dfa1.StateCount + dfa2augmented.StateCount) * (al.Count + 1);
int oldScirptSize = editScript.script.Count;
//Start with the internal script equals to null, at the end bestScript.Script will contain the best script
DFAEditScript bestScript = new DFAEditScript();
bestScript.script = null;
Stopwatch sw = new Stopwatch();
sw.Start();
// Iteratively check if there exists an edit of a given depth
for (int depth = 1; true; depth++)
{
var editList = new List <DFAEdit>();
if (GetDFAEditScriptTimeout(
dfa1, dfa2augmented, al, solver,
new List <long>(), editScript.script,
depth, timeout, sw, DFAUtilities.MyHillTestGeneration(al, dfa1, solver),
DFADensity.GetDFADensity(dfa1, al, solver),
editScript.GetCost(), bestScript, stateNamesMapping))
{
// if hits timeout break and return null
break;
}
if (bestScript.script != null)
{
bestScript.script.Reverse();
sw.Stop();
return(bestScript);
}
}
sw.Stop();
return(null);
}
