/*
* INITIALIZATION
* Create the projects for each submission, secret implemetation, and meta projects
* FileModifier.MakeProjects(topDir),
* FileModifier.MakeSecretProjects(topDir), FileModifier.MakeMetaProejcts(topDir)
* Build all the created projects
* BuildDirectory.BuildAllProjects(topDir, true), BuildDirectory.BuildAllSecreteProejcts(topDir, true);
* Check the project that cannot be compiled, the list will be ouput at the topDir.
* Manually process the list of projects before continuing.
* BuildDirectory.CheckNotBuiltProjects(topDir)
* Run Pex on all kind of the projects
* RunPex.RunPexOnSubmissionProjects(topDir), RunPex.RunPexOnSecretProjects(topDir),
* RunPex.RunPexOnMetaProjects(topDir)
* Extract tests from all kinds of the projects and write them to the xml file.
* ExtractPexResults
*
* METRIC 1 (Combined Path Counting)
* Run the tests of meta project on each meta projects, and get the test results that are failed
* M1 = |P_pass| / |P_all|
* Metric1.Compute()
*
* METRIC 2 (Separate Path Counting)
* Run the tests of secret project on both secret and submission projects, and compare the outputs that match (N)
* M2 = N / |P_secret|
* Metric2.Compute()
*
* METRIC 3 (Sample Inputs Counting)
* Randomly sample inputs (M) for each arguments, run these inputs on both secret and submission proejcts
* Compare the outputs that match (N)
* M3 = N / M
* Metric3.Compute()
*
*/
public static void Main(string[] args)
{
//string topDir = @"D:\Experiment\data-cleaner\apcs";
//string topDir = @"C:\Users\photocopy\Dropbox\Thac si\Luan van\Projects\Demo Paper\Pex4Fun\Pex4Fun\bin\Debug\Data";
string topDir = @"C:\Users\admin\Dropbox\Thac si\Luan van\Projects\Demo Paper\Pex4Fun\Pex4Fun\bin\Debug\Data";
FileModifier.MakeProjects(topDir);
FileModifier.MakeSecretProjects(topDir);
BuildDirectory.BuildProjects(topDir, true);
BuildDirectory.BuildSecretProjects(topDir, true);
FileModifier.MakeMetaProjects(topDir);
BuildDirectory.BuildMetaProjects(topDir, true);
BuildDirectory.CheckNotBuiltProjects(topDir);
RunPex.RunPexOnSecretProjects(topDir);
RunPex.RunPexOnMetaProjects(topDir);
ExtractPexResults.ExtractPexTests(topDir);
Metrics.ComputeMetric1(topDir);
Metrics.ComputeMetric2(topDir);
RandomTestGenerator.GenerateRandomTests(topDir, 200);
Metrics.ComputeMetric3(topDir);
Console.WriteLine(Utility.GetNumOfStudent(topDir));
Console.WriteLine(Utility.GetNumOfSubmissions(topDir));
//Console.ReadKey();
//List<string> notInclude = Utility.GetNotIncludedTasks(@"C:\Users\admin\Dropbox\Thac si\Luan van\Projects\Demo Paper\Pex4Fun\Pex4Fun\bin\Debug\Data");
//Utility.GenerateRandomlySampleSequence(topDir, notInclude);
//Utility.CopySampledSequences(@"D:\Experiment\apcs_sample.txt");
}
public static void CopySampledSequences(string sampleFile)
{
string[] files = File.ReadAllLines(sampleFile);
foreach (var line in files)
{
string destDir = line.Replace(@"D:\Experiment\data-cleaner\apcs", @"D:\Experiment\apcs_sample");
int end = line.LastIndexOf('\\');
string str = line.Substring(0, end);
string secretFile = str + @"\solution.cs";
end = str.LastIndexOf('\\');
str = str.Substring(end + 1);
string newSecretFile = @"D:\Experiment\apcs_sample\" + str + @"\solution.cs";
FileModifier.CopyDirectory(line, destDir);
File.Copy(secretFile, newSecretFile, true);
}
}
//12/03/18, By Jonathan
//This function extracts Pex results for a single project in the editedMetaProject folder
public static Tuple <HashSet <string[]>, HashSet <string[]>, string> ExtractTestsForEditedMetaProject(string editedMetaProject, string fileName, string className, string methodName, string assemblyName)
{
string assemblyFile = FileModifier.GetAssemblyFileOfProject(editedMetaProject, assemblyName);
MethodInfo method = RunTest.GetMethodDefinition(assemblyFile, className, methodName);
List <Test> tests = ExtractTestsForProject(editedMetaProject, method);
if (tests != null)
{
//Try to create "PexTests" directory. Nothing happens if already exists
System.IO.Directory.CreateDirectory(editedMetaProject + @"\PexTests");
string xmlFileName = editedMetaProject + @"\PexTests\" + fileName + "_PexTests.xml";
Serializer.SerializeTests(tests, xmlFileName);
}
//Notify caller of infin runtime / null ref exception
return(Tuple.Create(g_PassedTests, g_FailedTests, g_failType));
}
public static void GenerateRandomTests(string topDir, int numOfTests)
{
foreach (string taskDir in Directory.GetDirectories(topDir))
{
MethodInfo method = null;
foreach (var studentDir in Directory.GetDirectories(taskDir))
{
if (studentDir.EndsWith("secret_project"))
{
string assemblyFile = FileModifier.GetAssemblyFileOfProject(studentDir);
method = RunTest.GetMethodDefinition(assemblyFile, "Program", "Puzzle");
List <Test> tests = GenerateTestsForProject(method, numOfTests);
string fileName = studentDir + @"\RandomTests.xml";
Serializer.SerializeTests(tests, fileName);
break;
}
}
}
}
public static void ExtractPexTests(string topDir)
{
foreach (string taskDir in Directory.GetDirectories(topDir))
{
MethodInfo method = null;
foreach (var studentDir in Directory.GetDirectories(taskDir))
{
if (studentDir.EndsWith("secret_project"))
{
string assemblyFile = FileModifier.GetAssemblyFileOfProject(studentDir);
method = RunTest.GetMethodDefinition(assemblyFile, "Program", "Puzzle");
List <Test> tests = ExtractTestsForProject(studentDir, method);
if (tests != null)
{
string fileName = studentDir + @"\PexTests.xml";
Serializer.SerializeTests(tests, fileName);
}
break;
}
}
foreach (var studentDir in Directory.GetDirectories(taskDir))
{
if (studentDir.EndsWith("secret_project"))
{
continue;
}
foreach (var projectDir in Directory.GetDirectories(studentDir))
{
List <Test> tests = ExtractTestsForProject(projectDir, method);
if (tests != null)
{
string fileName = projectDir + @"\PexTests.xml";
Serializer.SerializeTests(tests, fileName);
}
}
}
}
}
//GENERAL NOTES (12/19/18):
// 1. MUST run Program.cs W/O debugging (Toolbar -> Debug -> Start w/o Debugging)
// 2. CANNOT have the ...\editedMetaProject\bin\Debug\reports folder open
public static void Main(string[] args)
{
string problemName = null, approach = "First", filterType = null, levelToRun = null;
if (args.Length == 4)
{
problemName = args[0];
approach = args[1];
filterType = args[2];
levelToRun = args[3];
}
else if (args.Length == 3)
{
problemName = args[0];
filterType = args[1];
levelToRun = args[2];
}
//Set that Pex4Fun.exe runs from as current directory
string currDir = System.IO.Directory.GetParent(Environment.CurrentDirectory).ToString();
//Move 4 levels up to set the top directory
//Go from C:...codesimilarity\Pex4Fun\Fa18_E2_120_3_countLessThan\behavioralsimilarity\Pex4Fun\bin\Debug\ -> C:...\codesimilarity\
string topDir = Path.GetFullPath(Path.Combine(currDir, @"..\..\..\..\..\..\"));
string editedMetaProjectDir = topDir + @"Approach\" + approach + @"\" + problemName + @"\editedMetaProject";
string allStudentsDir = topDir + @"Students\" + problemName;
string firstLevelResultsDir = topDir + @"Results\FirstLevel\" + problemName + @"\" + filterType;
string secondLevelResultsDir = topDir + @"Results\SecondLevel\Approach\CompleteEquivalence\" + problemName + @"\" + filterType;
string secondLevelResultsDirAlt = topDir + @"Results\SecondLevel\Approach\NewEquivalence\" + problemName + @"\" + filterType;
string subsumptionResultsDir = topDir + @"Results\SecondLevel\Approach\Subsumption\" + problemName;
string assemblyName = "meta_project1525456207";
string hardcodedMetaProjName = "meta_project1525456207";
string instructorSolnDir = topDir + @"InstructorSolutions\" + problemName + ".cs";
string className = "Partitioner";
Random randNum = new Random();
Dictionary <double, HashSet <string> > uniquePassingTestsClusterMap = new Dictionary <double, HashSet <string> >();
Dictionary <double, HashSet <string> > uniqueFailingTestsClusterMap = new Dictionary <double, HashSet <string> >();
//Dictionary key maps to List -> List -> string arrays...lets me have mult. vals. map to single key
Dictionary <double, List <List <string[]> > > scoreToTestBodyMap = new Dictionary <double, List <List <string[]> > >();
/* Cluster level one */
if (levelToRun.ToLower() == "first" || levelToRun.ToLower() == "both")
{
string[] allStudentDirs_partition = Directory.GetDirectories(allStudentsDir);
//Get all files across allStudentDirs
var filesArr = Directory.EnumerateDirectories(allStudentsDir).SelectMany(directory
=> Directory.EnumerateFiles(directory, "*.cs"));
//Conv arr -> List for easier manipulation
List <string> filesList = new List <string>(filesArr);
for (int i = 0; i < filesList.Count; i++)
{
string partitionFile = filesList[i];
if (partitionFile.EndsWith(".cs"))
{
//Below false IFF empty addToEnd method body
if (FileModifier.CopyBodyToMetaProgram(partitionFile, editedMetaProjectDir, className, hardcodedMetaProjName))
{
//Account for default / non-default Pex emission filter
changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
{
//Below line ensures that Pex4Fun uses most updated ver. of asembly file
copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);
string fileName = partitionFile.Substring(partitionFile.LastIndexOf("\\") + 1);
int lastIdx = partitionFile.LastIndexOf('\\');
string tmp = partitionFile.Substring(0, lastIdx);
string fileDirName = tmp.Substring(tmp.LastIndexOf('\\') + 1);
//12/03/18, by Jonathan: Method to run pex on editedMetaProject's MetaProgram.cs
RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
Tuple <HashSet <string[]>, HashSet <string[]>, string> passedAndFailedTests = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, fileDirName, "MetaProgram", "Check", assemblyName);
HashSet <string[]> passedTests = passedAndFailedTests.Item1;
HashSet <string[]> failedTests = passedAndFailedTests.Item2;
//Round score to nearest hundreth
double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, fileDirName), 2);
//string moveFileToClusterFileName = fileName;
string moveFileToClusterScore = score.ToString();
string recordPassedMethodBodyFilename = score.ToString();
bool areTestsIdentical = false;
int clusterNum = 1;
//If there were 1+ passing tests...
if (passedTests.Count != 0)
{
List <List <string[]> > existantTestBodiesList = new List <List <string[]> >();
//FIRST check if key already exists in Dictionary before adding...
if (scoreToTestBodyMap.TryGetValue(score, out existantTestBodiesList))
{
//Check if both lists passed on SAME tests, regardless of order / dupe entries
var list1Lookup = new HashSet <string[]>(passedTests);
foreach (var methodBodies in existantTestBodiesList)
{
var list2Lookup = new HashSet <string[]>(methodBodies);
if (list1Lookup.SetEquals(list2Lookup))
{
//If they've got same values, then they truly belong in same parent cluster
recordPassedMethodBodyFilename = score.ToString();
moveFileToClusterScore = score.ToString();
areTestsIdentical = true;
break;
}
else
{
//Else they belong in same parent cluster, but DIFF child clusters
clusterNum++;
moveFileToClusterScore = score.ToString() + @"\" + score.ToString() + "_" + clusterNum.ToString();
recordPassedMethodBodyFilename = score.ToString() + @"\" + clusterNum.ToString();
}
}
}
//Else if the key didn't already exist in Dictionary...
//...Store the list in memory IFF tests are NOT identical...
if (!areTestsIdentical)
{
//If already contains 'score' key, just append the test to it!
if (!scoreToTestBodyMap.ContainsKey(score))
{
scoreToTestBodyMap.Add(score, new List <List <string[]> >());
}
scoreToTestBodyMap[score].Add(passedTests.ToList());
}
//...Store list in file
recordPassedAndFailedMethodBodies(fileDirName, passedTests, failedTests, firstLevelResultsDir,
recordPassedMethodBodyFilename);
}
moveDirToCluster(moveFileToClusterScore, firstLevelResultsDir, allStudentDirs_partition[i], fileName);
//recordLevelOneResult(score, studentDir, fileName);
}
}
//MetaProgram.cs didn't build
else
{
//moveSubmissionToUnsuccessful(studentDir, file, null);
}
}
}
}
/* Cluster level 2 */
if (approach.ToLower() != "subsumption" && (levelToRun.ToLower() == "second" || levelToRun.ToLower() == "both"))
{
string chosenDir = firstLevelResultsDir + "/0";
Dictionary <string, double> submissionScoreMap = new Dictionary <string, double>();
string[] allStudentDirs_partition = Directory.GetDirectories(chosenDir);
//Get all files across allStudentDirs
var filesArr = Directory.EnumerateDirectories(chosenDir).SelectMany(directory
=> Directory.EnumerateFiles(directory, "*.cs"));
//Conv arr -> List for easier manipulation
List <string> filesList = new List <string>(filesArr);
//Insert instructor correct solution at beginning of list (first pass = comparison to ACTUAL soln)
filesList.Insert(0, instructorSolnDir);
for (int i = 0; i < filesList.Count - 1; i++)
{
string solutionFile = filesList[i];
string solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);
if (solutionFile.EndsWith(".cs"))
{
//Establish the current file as the ref soln (SubmissionB)
if (FileModifier.replaceSolutionMethodBody(className, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
{
int fileCount = filesArr.Count();
//Pair-wise comparison on each file
for (int j = i + 1; j < filesList.Count; j++)
{
string submissionFile = filesList[j];
if (submissionFile.EndsWith(".cs"))
{
string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);
int lastIndx = submissionFile.LastIndexOf('\\');
string temp = submissionFile.Substring(0, lastIndx);
string submissionDirName = temp.Substring(temp.LastIndexOf('\\') + 1);
lastIndx = filesList[i].LastIndexOf('\\');
temp = filesList[i].Substring(0, lastIndx);
string solutionDirName = temp.Substring(temp.LastIndexOf('\\') + 1);
if (fileCount == 1)
{
recordLevelTwoResult(firstLevelResultsDir, "Cluster: " + solutionDirName
+ "\tReference Submission: " + submissionFileName + "\tMatched");
}
else
{
//Establish submissionA
//Return false if unsuccessful copy (empty method body)
if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, className, hardcodedMetaProjName))
{
//If compiled correctly after having replaced the body
changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
{
//Below line ensures that Pex4Fun uses most updated ver. of asembly file
copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);
RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
//string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
//string solutionFileNameMinusCS = solutionFileName.Substring(0, solutionFileName.Length - 3);
string combinedFileName = submissionDirName + "&" + solutionDirName;
Tuple <HashSet <string[]>, HashSet <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
//Item1 = passedTests, Item2 = failedTests
//string failType = extractedResults.Item3;
//if failType == "assert"... else if failType == "exception"...
//Round score to nearest hundreth
double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, combinedFileName), 2);
recordPassedAndFailedMethodBodies(combinedFileName, extractedResults.Item1, extractedResults.Item2, secondLevelResultsDir, score.ToString());
//TODO:
// Use FIRST pass score to figure out what initial cluster to assign folks to, THEN use 1/0 to figure out whether submissions should be clustered together
//First pass, add all scores to the
//Create mapping of PSE scores assigned during first pass
if (i == 0)
{
//Add unique passing tests to map
if (!uniquePassingTestsClusterMap.ContainsKey(score))
{
uniquePassingTestsClusterMap.Add(score, new HashSet <string>());
}
//Conv string[] -> string, preserving "\n"
uniquePassingTestsClusterMap[score].Add(string.Join("", extractedResults.Item1.Select(x => string.Join("", x))));
//Add unique failing tests to map
if (!uniqueFailingTestsClusterMap.ContainsKey(score))
{
uniqueFailingTestsClusterMap.Add(score, new HashSet <string>());
}
//Conv string[] -> string
uniqueFailingTestsClusterMap[score].Add(string.Join("", extractedResults.Item2.Select(x => string.Join("", x))));
//Add solution name and score to map
if (!submissionScoreMap.ContainsKey(solutionFileName))
{
submissionScoreMap.Add(solutionFileName, score);
}
submissionScoreMap[solutionFileName] = score;
//Add submission name and score to map
if (!submissionScoreMap.ContainsKey(submissionFileName))
{
submissionScoreMap.Add(submissionFileName, score);
}
submissionScoreMap[submissionFileName] = score;
}
if (score == 1)
{
recordLevelTwoResult(firstLevelResultsDir, "Cluster: " + solutionDirName
+ "\tResultant PSE Score: " + score
+ "\tReference Submission: " + solutionDirName
+ "\tCompared Submission: " + submissionDirName
+ "\tMatched");
//Add tests pertaining to PSE score assigned during first pass here:
if (i != 0)
{
score = submissionScoreMap[submissionFileName];
}
//Add unique passing tests to map
if (!uniquePassingTestsClusterMap.ContainsKey(score))
{
uniquePassingTestsClusterMap.Add(score, new HashSet <string>());
}
//Conv string[] -> string, preserving "\n"
uniquePassingTestsClusterMap[score].Add(string.Join("", extractedResults.Item1.Select(x => string.Join("", x))));
//Add unique failing tests to map
if (!uniqueFailingTestsClusterMap.ContainsKey(score))
{
uniqueFailingTestsClusterMap.Add(score, new HashSet <string>());
}
uniqueFailingTestsClusterMap[score].Add(string.Join("", extractedResults.Item2.Select(x => string.Join("", x))));
//Cluster those that got matched and remove them from filesList
filesList.RemoveAt(j);
j--; //Keep inner ptr in same place after removal
Console.WriteLine(filesList[j]);
}
else
{
recordLevelTwoResult(firstLevelResultsDir, "Cluster: " + solutionDirName
+ "\tResultant PSE Score: " + score
+ "\tReference Submission: " + solutionDirName
+ "\tCompared Submission: " + submissionDirName
+ "\tNot Matched");
}
}
}
//MetaProgram.cs didn't build
else
{
//moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
Console.WriteLine(solutionDirName + " and " + submissionDirName + " failed to produce compiling editedMetaProject");
}
}
}
}
}
}
}
}
//TODO: Keep one test as the reference. Loop over all other tests and call the Except() to figure out the distinct set of tests that the reference has
//Q from Angello: How to ensure that the unique, minimal test set we're generating per cluster PASSES on every other cluster but fails its own cluster??
foreach (var reference in uniquePassingTestsClusterMap)
{
//Initialize IEnumerable<string> as the first test in dict
var currTest = uniquePassingTestsClusterMap[reference.Key].Intersect(uniquePassingTestsClusterMap[reference.Key]);
foreach (var other in uniquePassingTestsClusterMap)
{
if (reference.Key == other.Key)
{
continue;
}
//After looping done, currTest is the MIN set of passing tests to differentiate cluster from others
currTest = currTest.Intersect(uniquePassingTestsClusterMap[reference.Key]
.Except(uniquePassingTestsClusterMap[other.Key]));
}
recordUniquePassFailClusterTests(currTest, reference.Key, secondLevelResultsDir, "Passed");
}
foreach (var reference in uniqueFailingTestsClusterMap)
{
var currTest = uniqueFailingTestsClusterMap[reference.Key].Intersect(uniqueFailingTestsClusterMap[reference.Key]);
foreach (var other in uniqueFailingTestsClusterMap)
{
if (reference.Key == other.Key)
{
continue;
}
//After looping done, currTest is the MIN set of failing tests to differentiate cluster from others
currTest = currTest.Intersect(uniqueFailingTestsClusterMap[reference.Key]
.Except(uniqueFailingTestsClusterMap[other.Key]));
}
recordUniquePassFailClusterTests(currTest, reference.Key, secondLevelResultsDir, "Failed");
}
/* Level 2: Subsumption */
if (approach.ToLower() == "subsumption")
{
string chosenDir = secondLevelResultsDir;
if (!Directory.Exists(secondLevelResultsDir))
{
chosenDir = secondLevelResultsDirAlt; //Choose alternate secondLevelDir if orig. doesn't exist!
}
foreach (var pseScoreFolder in Directory.GetDirectories(chosenDir))
{
Console.WriteLine("Evaluating PSE FOLDER: " + pseScoreFolder.Substring(pseScoreFolder.LastIndexOf("\\") + 1));
string[] allChildClusters = Directory.GetDirectories(pseScoreFolder);
for (int i = 0; i < allChildClusters.Length - 1; i++)
{
string currChild = allChildClusters[i];
//Skip the PSE = 1 and PSE = NaN folders
if (currChild.Substring(currChild.LastIndexOf("\\") + 1) == "1" ||
currChild.Contains("NaN"))
{
continue;
}
string[] solutionFilesList = Directory.GetFiles(currChild); //Get all files within currChild
Random r = new Random();
int randIndx = r.Next(0, solutionFilesList.Length); //[0, filesList.Length)
string solutionFile = solutionFilesList[randIndx]; //Ref Soln = rand file inside child
string solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);
if (solutionFile.EndsWith(".cs"))
{
//Establish the current file as the ref soln
if (FileModifier.replaceSolutionMethodBody(className, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
{
string nextChild = allChildClusters[i + 1]; //Get next child
if (nextChild.Substring(nextChild.LastIndexOf("\\") + 1) == "1" ||
nextChild.Contains("NaN"))
{
continue;
}
string[] submissionFilesList = Directory.GetFiles(nextChild); //Get all files within nextChild
randIndx = r.Next(0, submissionFilesList.Length); //[0, filesList.Length)
string submissionFile = submissionFilesList[randIndx]; //Ref Soln = rand file inside child
string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);
if (submissionFile.EndsWith(".cs"))
{
Console.WriteLine("Both " + solutionFileName + " and " + submissionFileName + " end in .cs");
//Begin replacing method body of submission
//Return false if unsuccessful copy (empty method body)
if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, className, hardcodedMetaProjName))
{
//If compiled correctly after having replaced the body
changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
{
//Below line ensures that Pex4Fun uses most updated ver. of asembly file
copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);
RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
string solutionFileNameMinusCS = solutionFileName.Substring(0, solutionFileName.Length - 3);
string combinedFileName = solutionFileNameMinusCS + "&" + submissionFileNameMinusCS;
Tuple <HashSet <string[]>, HashSet <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
string failType = extractedResults.Item3;
string message = "";
if (failType == "assertion")
{
Console.WriteLine("s2 >= s1");
message = "Reference solution subsumes student submission.";
}
else if (failType == "exception")
{
Console.WriteLine("s1 >= s2");
message = "Student submission subsumes reference solution.";
}
else if (failType == "")
{
message = "No subsumption.";
}
else
{
message = "Neither...something likely went wrong.";
}
//Record to file in subsumption folder
recordSubsumptionResult(subsumptionResultsDir, combinedFileName, message);
}
//MetaProgram.cs didn't build
else
{
//moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
Console.WriteLine(solutionFileName + " and " + submissionFileName + " failed to produce compiling editedMetaProject");
}
}
}
}
}
}
}
}
} //End Main()
//GENERAL NOTES (12/19/18):
// 1. MUST run Program.cs W/O debugging (Toolbar -> Debug -> Start w/o Debugging)
// 2. CANNOT have the ...\editedMetaProject\bin\Debug\reports folder open
public static void Main(string[] args)
{
string problemName = args[0];
string approach = args[1];
string filterType = args[2];
string levelToRun = args[3];
//Set that Pex4Fun.exe runs from as current directory
string currDir = System.IO.Directory.GetParent(Environment.CurrentDirectory).ToString();
//Move 4 levels up to set the top directory
//Go from C:...codesimilarity\Pex4Fun\Sp18_Q10_20_sorted\behavioralsimilarity\Pex4Fun\bin\Debug\ -> C:...\codesimilarity\
string topDir = Path.GetFullPath(Path.Combine(currDir, @"..\..\..\..\..\..\"));
string editedMetaProjectDir = topDir + @"Approach\" + approach + @"\" + problemName + @"\editedMetaProject";
string allStudentsDir = topDir + @"Students\" + problemName;
string firstLevelResultsDir = topDir + @"Results\FirstLevel\" + problemName + @"\" + filterType;
string secondLevelResultsDir = topDir + @"Results\SecondLevel\Approach\CompleteEquivalence\" + problemName + @"\" + filterType;
string secondLevelResultsDirAlt = topDir + @"Results\SecondLevel\Approach\NewEquivalence\" + problemName + @"\" + filterType;
string subsumptionResultsDir = topDir + @"Results\SecondLevel\Approach\Subsumption\" + problemName;
string assemblyName = "meta_project1525456207";
string hardcodedMetaProjName = "meta_project1525456207";
string methodName = "sort";
Random randNum = new Random();
//Dictionary key maps to List -> List -> string arrays...lets me have mult. vals. map to single key
Dictionary <double, List <List <string[]> > > scoreToTestBodyMap = new Dictionary <double, List <List <string[]> > >();
/* Cluster level one */
if (levelToRun.ToLower() == "first" || levelToRun.ToLower() == "both")
{
//Choose each folder that ends in "@illinois.edu"
foreach (var studentDir in Directory.GetDirectories(allStudentsDir))
{
if (studentDir.Contains("@illinois.edu"))
{
foreach (var file in Directory.GetFiles(studentDir))
{
if (file.EndsWith(".cs"))
{
//Below false IFF empty addToEnd method body
if (FileModifier.CopyBodyToMetaProgram(file, editedMetaProjectDir, methodName, hardcodedMetaProjName))
{
//Account for default / non-default Pex emission filter
changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
{
//Below line ensures that Pex4Fun uses most updated ver. of asembly file
copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);
string fileName = file.Substring(file.LastIndexOf("\\") + 1);
//12/03/18, by Jonathan: Method to run pex on editedMetaProject's MetaProgram.cs
RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
string fileNameMinusCS = fileName.Substring(0, fileName.Length - 3);
Tuple <List <string[]>, List <string[]>, string> passedAndFailedTests = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, fileNameMinusCS, "MetaProgram", "Check", assemblyName);
List <string[]> passedTests = passedAndFailedTests.Item1;
List <string[]> failedTests = passedAndFailedTests.Item2;
//Round score to nearest hundreth
double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, fileNameMinusCS), 2);
//string moveFileToClusterFileName = fileName;
string moveFileToClusterScore = score.ToString();
string recordPassedMethodBodyFilename = score.ToString();
bool areTestsIdentical = false;
//If there were 1+ passing tests...
if (passedTests.Count != 0)
{
List <List <string[]> > existantTestBodiesList = new List <List <string[]> >();
//FIRST check if key already exists in Dictionary before adding...
if (scoreToTestBodyMap.TryGetValue(score, out existantTestBodiesList))
{
//Check if both lists passed on SAME tests, regardless of order / dupe entries
var list1Lookup = new HashSet <string[]>(passedTests);
foreach (var methodBodies in existantTestBodiesList)
{
var list2Lookup = new HashSet <string[]>(methodBodies);
if (list1Lookup.SetEquals(list2Lookup))
{
//If they've got same values, then they truly belong in same parent cluster
recordPassedMethodBodyFilename = score.ToString();
moveFileToClusterScore = score.ToString();
areTestsIdentical = true;
break;
}
else
{
//Else they belong in same parent cluster, but DIFF child clusters
string randNumStr = randNum.Next(0, 100).ToString();
moveFileToClusterScore = score.ToString() + "_" + randNumStr;
recordPassedMethodBodyFilename = score.ToString() + @"\" + randNumStr;
}
}
}
//Else if the key didn't already exist in Dictionary...
//...Store the list in memory IFF tests are NOT identical...
if (!areTestsIdentical)
{
scoreToTestBodyMap.Add(score, new List <List <string[]> >());
scoreToTestBodyMap[score].Add(passedTests);
}
//...Store list in file
recordPassedAndFailedMethodBodies(passedTests, failedTests, firstLevelResultsDir,
recordPassedMethodBodyFilename);
}
moveFileToCluster(moveFileToClusterScore, firstLevelResultsDir, file, fileName);
//recordLevelOneResult(score, studentDir, fileName);
}
}
//MetaProgram.cs didn't build
else
{
//moveSubmissionToUnsuccessful(studentDir, file, null);
}
}
}
}
}
}
/* Cluster level 2 */
if (approach.ToLower() != "subsumption" && (levelToRun.ToLower() == "second" || levelToRun.ToLower() == "both"))
{
//Skip the PSE = 1 and PSE = NaN folders
foreach (var pseScoreFolder in Directory.GetDirectories(firstLevelResultsDir))
{
if (pseScoreFolder.Substring(pseScoreFolder.LastIndexOf("\\") + 1) == "1" ||
pseScoreFolder.Contains("NaN") || pseScoreFolder.Contains("Passed&FailedTests"))
{
continue;
}
int solutionFileIndex = 0;
string[] filesList = Directory.GetFiles(pseScoreFolder);
foreach (var solutionFile in filesList)
{
string solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);
if (solutionFile.EndsWith(".cs"))
{
//Establish the current file as the ref soln
if (FileModifier.replaceSolutionMethodBody(methodName, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
{
int fileCount = Directory.GetFiles(pseScoreFolder).Length;
int submissionFileIndex = 0;
//Pair-wise comparison on each file
foreach (var submissionFile in filesList)
{
if (submissionFile.EndsWith(".cs"))
{
string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);
string pseScoreFolderName = pseScoreFolder.Substring(pseScoreFolder.LastIndexOf('\\') + 1);
if (fileCount == 1)
{
recordLevelTwoResult(pseScoreFolder, "PSE Cluster: " + pseScoreFolderName
+ "\tReference Submission: " + submissionFileName + "\tMatched");
}
else if (submissionFileIndex > solutionFileIndex)
{
//Begin replacing method body of submission
//Return false if unsuccessful copy (empty method body)
if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, methodName, hardcodedMetaProjName))
{
//If compiled correctly after having replaced the body
changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
{
//Below line ensures that Pex4Fun uses most updated ver. of asembly file
copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);
RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
string solutionFileNameMinusCS = solutionFileName.Substring(0, solutionFileName.Length - 3);
string combinedFileName = solutionFileNameMinusCS + "&" + submissionFileNameMinusCS;
Tuple <List <string[]>, List <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
//string failType = extractedResults.Item3;
//if failType == "assert"... else if failType == "exception"...
//Round score to nearest hundreth
double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, combinedFileName), 2);
if (score == 1)
{
recordLevelTwoResult(pseScoreFolder, "PSE Cluster: " + pseScoreFolderName
+ "\tResultant PSE Score: " + score
+ "\tReference Submission: " + solutionFileName
+ "\tCompared Submission: " + submissionFileName
+ "\tMatched");
//Remove submissionFile from filesList if score == 1 (more efficient pairwise algorithm)
filesList = filesList.Where(w => w != filesList[submissionFileIndex]).ToArray();
submissionFileIndex--; //Decrem then increm submissionFileIndex to keep value the same
}
else
{
recordLevelTwoResult(pseScoreFolder, "PSE Cluster: " + pseScoreFolderName
+ "\tResultant PSE Score: " + score
+ "\tReference Submission: " + solutionFileName
+ "\tCompared Submission: " + submissionFileName
+ "\tNot Matched");
}
}
}
//MetaProgram.cs didn't build
else
{
//moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
Console.WriteLine(solutionFileName + " and " + submissionFileName + " failed to produce compiling editedMetaProject");
}
}
}
submissionFileIndex++;
}
}
}
solutionFileIndex++;
}
}
}
/* Level 2: Subsumption */
if (approach.ToLower() == "subsumption")
{
string chosenDir = secondLevelResultsDir;
if (!Directory.Exists(secondLevelResultsDir))
{
chosenDir = secondLevelResultsDirAlt; //Choose alternate secondLevelDir if orig. doesn't exist!
}
string[] allParentClusters = Directory.GetDirectories(chosenDir);
for (int j = 0; j < allParentClusters.Length; j++)
{
string pseScoreFolder = allParentClusters[j];
Console.WriteLine("Evaluating PSE FOLDER: " + pseScoreFolder.Substring(pseScoreFolder.LastIndexOf("\\") + 1));
string[] allChildClusters = Directory.GetDirectories(pseScoreFolder);
for (int i = 0; i < allChildClusters.Length; i++)
{
string currChild = allChildClusters[i];
Console.WriteLine("CURRCHILD = " + currChild);
//Skip the PSE = 1 and PSE = NaN folders
if (currChild.Substring(currChild.LastIndexOf("\\") + 1) == "1" ||
currChild.Contains("NaN"))
{
continue;
}
string[] solutionFilesList = Directory.GetFiles(currChild); //Get all files within currChild
Random r = new Random();
int randIndx = r.Next(0, solutionFilesList.Length); //[0, filesList.Length)
string solutionFile = solutionFilesList[randIndx]; //Ref Soln = rand file inside child
string solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);
if (solutionFile.EndsWith(".cs"))
{
//Establish the current file as the ref soln
if (FileModifier.replaceSolutionMethodBody(methodName, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
{
//TO EDIT (04/05/2019): Need to select ONE other file from each child cluster inside...
//...DIFF parent cluster!!
for (int k = j + 1; k < allParentClusters.Length; k++)
{
string nextPseScoreFolder = allParentClusters[k];
string[] allNextChildClusters = Directory.GetDirectories(nextPseScoreFolder);
for (int x = 0; x < allNextChildClusters.Length; x++)
{
string nextChild = allChildClusters[x]; //Get next child
if (nextChild.Substring(nextChild.LastIndexOf("\\") + 1) == "1" ||
nextChild.Contains("NaN"))
{
continue;
}
string[] submissionFilesList = Directory.GetFiles(nextChild); //Get all files within nextChild
randIndx = r.Next(0, submissionFilesList.Length); //[0, filesList.Length)
string submissionFile = submissionFilesList[randIndx]; //Ref Soln = rand file inside child
string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);
if (submissionFile.EndsWith(".cs"))
{
Console.WriteLine("Both " + solutionFileName + " and " + submissionFileName + " end in .cs");
//Begin replacing method body of submission
//Return false if unsuccessful copy (empty method body)
if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, methodName, hardcodedMetaProjName))
{
//If compiled correctly after having replaced the body
changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
{
//Below line ensures that Pex4Fun uses most updated ver. of asembly file
copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);
RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
string solutionFileNameMinusCS = solutionFileName.Substring(0, solutionFileName.Length - 3);
string combinedFileName = solutionFileNameMinusCS + "&" + submissionFileNameMinusCS;
Tuple <List <string[]>, List <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
string failType = extractedResults.Item3;
string message = "";
if (failType == "assertion")
{
Console.WriteLine("s2 >= s1");
message = "Reference solution subsumes student submission.";
}
else if (failType == "exception")
{
Console.WriteLine("s1 >= s2");
message = "Student submission subsumes reference solution.";
}
else if (failType == "")
{
message = "No subsumption.";
}
else
{
message = "Neither...something likely went wrong.";
}
//Record to file in subsumption folder
recordSubsumptionResult(subsumptionResultsDir, combinedFileName, message);
}
//MetaProgram.cs didn't build
else
{
//moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
Console.WriteLine(solutionFileName + " and " + submissionFileName + " failed to produce compiling editedMetaProject");
}
}
}
}
}
}
}
}
}
}
} //End Main()
