Beispiel #1
0
        /*
         * INITIALIZATION
         * Create the projects for each submission, secret implemetation, and meta projects
         *      FileModifier.MakeProjects(topDir),
         *      FileModifier.MakeSecretProjects(topDir), FileModifier.MakeMetaProejcts(topDir)
         * Build all the created projects
         *      BuildDirectory.BuildAllProjects(topDir, true), BuildDirectory.BuildAllSecreteProejcts(topDir, true);
         * Check the project that cannot be compiled, the list will be ouput at the topDir.
         * Manually process the list of projects before continuing.
         *      BuildDirectory.CheckNotBuiltProjects(topDir)
         * Run Pex on all kind of the projects
         *      RunPex.RunPexOnSubmissionProjects(topDir), RunPex.RunPexOnSecretProjects(topDir),
         *      RunPex.RunPexOnMetaProjects(topDir)
         * Extract tests from all kinds of the projects and write them to the xml file.
         *      ExtractPexResults
         *
         * METRIC 1 (Combined Path Counting)
         * Run the tests of meta project on each meta projects, and get the test results that are failed
         * M1 = |P_pass| / |P_all|
         *      Metric1.Compute()
         *
         * METRIC 2 (Separate Path Counting)
         * Run the tests of secret project on both secret and submission projects, and compare the outputs that match (N)
         * M2 = N / |P_secret|
         *      Metric2.Compute()
         *
         * METRIC 3 (Sample Inputs Counting)
         * Randomly sample inputs (M) for each arguments, run these inputs on both secret and submission proejcts
         * Compare the outputs that match (N)
         * M3 = N / M
         *      Metric3.Compute()
         *
         */

        public static void Main(string[] args)
        {
            //string topDir = @"D:\Experiment\data-cleaner\apcs";
            //string topDir = @"C:\Users\photocopy\Dropbox\Thac si\Luan van\Projects\Demo Paper\Pex4Fun\Pex4Fun\bin\Debug\Data";
            string topDir = @"C:\Users\admin\Dropbox\Thac si\Luan van\Projects\Demo Paper\Pex4Fun\Pex4Fun\bin\Debug\Data";

            FileModifier.MakeProjects(topDir);
            FileModifier.MakeSecretProjects(topDir);

            BuildDirectory.BuildProjects(topDir, true);
            BuildDirectory.BuildSecretProjects(topDir, true);

            FileModifier.MakeMetaProjects(topDir);
            BuildDirectory.BuildMetaProjects(topDir, true);
            BuildDirectory.CheckNotBuiltProjects(topDir);

            RunPex.RunPexOnSecretProjects(topDir);
            RunPex.RunPexOnMetaProjects(topDir);

            ExtractPexResults.ExtractPexTests(topDir);

            Metrics.ComputeMetric1(topDir);
            Metrics.ComputeMetric2(topDir);

            RandomTestGenerator.GenerateRandomTests(topDir, 200);
            Metrics.ComputeMetric3(topDir);

            Console.WriteLine(Utility.GetNumOfStudent(topDir));
            Console.WriteLine(Utility.GetNumOfSubmissions(topDir));
            //Console.ReadKey();
            //List<string> notInclude = Utility.GetNotIncludedTasks(@"C:\Users\admin\Dropbox\Thac si\Luan van\Projects\Demo Paper\Pex4Fun\Pex4Fun\bin\Debug\Data");
            //Utility.GenerateRandomlySampleSequence(topDir, notInclude);
            //Utility.CopySampledSequences(@"D:\Experiment\apcs_sample.txt");
        }
Beispiel #2
0
        //GENERAL NOTES (12/19/18):
        // 1. MUST run Program.cs W/O debugging (Toolbar -> Debug -> Start w/o Debugging)
        // 2. CANNOT have the ...\editedMetaProject\bin\Debug\reports folder open

        public static void Main(string[] args)
        {
            string problemName = null, approach = "First", filterType = null, levelToRun = null;

            if (args.Length == 4)
            {
                problemName = args[0];
                approach    = args[1];
                filterType  = args[2];
                levelToRun  = args[3];
            }
            else if (args.Length == 3)
            {
                problemName = args[0];
                filterType  = args[1];
                levelToRun  = args[2];
            }

            //Set that Pex4Fun.exe runs from as current directory
            string currDir = System.IO.Directory.GetParent(Environment.CurrentDirectory).ToString();
            //Move 4 levels up to set the top directory
            //Go from C:...codesimilarity\Pex4Fun\Fa18_E2_120_3_countLessThan\behavioralsimilarity\Pex4Fun\bin\Debug\ -> C:...\codesimilarity\
            string topDir = Path.GetFullPath(Path.Combine(currDir, @"..\..\..\..\..\..\"));
            string editedMetaProjectDir     = topDir + @"Approach\" + approach + @"\" + problemName + @"\editedMetaProject";
            string allStudentsDir           = topDir + @"Students\" + problemName;
            string firstLevelResultsDir     = topDir + @"Results\FirstLevel\" + problemName + @"\" + filterType;
            string secondLevelResultsDir    = topDir + @"Results\SecondLevel\Approach\CompleteEquivalence\" + problemName + @"\" + filterType;
            string secondLevelResultsDirAlt = topDir + @"Results\SecondLevel\Approach\NewEquivalence\" + problemName + @"\" + filterType;
            string subsumptionResultsDir    = topDir + @"Results\SecondLevel\Approach\Subsumption\" + problemName;
            string assemblyName             = "meta_project1525456207";
            string hardcodedMetaProjName    = "meta_project1525456207";
            string instructorSolnDir        = topDir + @"InstructorSolutions\" + problemName + ".cs";
            string className = "Partitioner";
            Random randNum   = new Random();
            Dictionary <double, HashSet <string> > uniquePassingTestsClusterMap = new Dictionary <double, HashSet <string> >();
            Dictionary <double, HashSet <string> > uniqueFailingTestsClusterMap = new Dictionary <double, HashSet <string> >();


            //Dictionary key maps to List -> List -> string arrays...lets me have mult. vals. map to single key
            Dictionary <double, List <List <string[]> > > scoreToTestBodyMap = new Dictionary <double, List <List <string[]> > >();


            /* Cluster level one */
            if (levelToRun.ToLower() == "first" || levelToRun.ToLower() == "both")
            {
                string[] allStudentDirs_partition = Directory.GetDirectories(allStudentsDir);
                //Get all files across allStudentDirs
                var filesArr = Directory.EnumerateDirectories(allStudentsDir).SelectMany(directory
                                                                                         => Directory.EnumerateFiles(directory, "*.cs"));
                //Conv arr -> List for easier manipulation
                List <string> filesList = new List <string>(filesArr);
                for (int i = 0; i < filesList.Count; i++)
                {
                    string partitionFile = filesList[i];
                    if (partitionFile.EndsWith(".cs"))
                    {
                        //Below false IFF empty addToEnd method body
                        if (FileModifier.CopyBodyToMetaProgram(partitionFile, editedMetaProjectDir, className, hardcodedMetaProjName))
                        {
                            //Account for default / non-default Pex emission filter
                            changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
                            if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
                            {
                                //Below line ensures that Pex4Fun uses most updated ver. of asembly file
                                copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);

                                string fileName    = partitionFile.Substring(partitionFile.LastIndexOf("\\") + 1);
                                int    lastIdx     = partitionFile.LastIndexOf('\\');
                                string tmp         = partitionFile.Substring(0, lastIdx);
                                string fileDirName = tmp.Substring(tmp.LastIndexOf('\\') + 1);

                                //12/03/18, by Jonathan: Method to run pex on editedMetaProject's MetaProgram.cs
                                RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
                                Tuple <HashSet <string[]>, HashSet <string[]>, string> passedAndFailedTests = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, fileDirName, "MetaProgram", "Check", assemblyName);
                                HashSet <string[]> passedTests = passedAndFailedTests.Item1;
                                HashSet <string[]> failedTests = passedAndFailedTests.Item2;
                                //Round score to nearest hundreth
                                double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, fileDirName), 2);

                                //string moveFileToClusterFileName = fileName;
                                string moveFileToClusterScore         = score.ToString();
                                string recordPassedMethodBodyFilename = score.ToString();
                                bool   areTestsIdentical = false;
                                int    clusterNum        = 1;

                                //If there were 1+ passing tests...
                                if (passedTests.Count != 0)
                                {
                                    List <List <string[]> > existantTestBodiesList = new List <List <string[]> >();
                                    //FIRST check if key already exists in Dictionary before adding...
                                    if (scoreToTestBodyMap.TryGetValue(score, out existantTestBodiesList))
                                    {
                                        //Check if both lists passed on SAME tests, regardless of order / dupe entries
                                        var list1Lookup = new HashSet <string[]>(passedTests);
                                        foreach (var methodBodies in existantTestBodiesList)
                                        {
                                            var list2Lookup = new HashSet <string[]>(methodBodies);
                                            if (list1Lookup.SetEquals(list2Lookup))
                                            {
                                                //If they've got same values, then they truly belong in same parent cluster
                                                recordPassedMethodBodyFilename = score.ToString();
                                                moveFileToClusterScore         = score.ToString();
                                                areTestsIdentical = true;
                                                break;
                                            }
                                            else
                                            {
                                                //Else they belong in same parent cluster, but DIFF child clusters
                                                clusterNum++;
                                                moveFileToClusterScore         = score.ToString() + @"\" + score.ToString() + "_" + clusterNum.ToString();
                                                recordPassedMethodBodyFilename = score.ToString() + @"\" + clusterNum.ToString();
                                            }
                                        }
                                    }
                                    //Else if the key didn't already exist in Dictionary...
                                    //...Store the list in memory IFF tests are NOT identical...
                                    if (!areTestsIdentical)
                                    {
                                        //If already contains 'score' key, just append the test to it!
                                        if (!scoreToTestBodyMap.ContainsKey(score))
                                        {
                                            scoreToTestBodyMap.Add(score, new List <List <string[]> >());
                                        }
                                        scoreToTestBodyMap[score].Add(passedTests.ToList());
                                    }
                                    //...Store list in file
                                    recordPassedAndFailedMethodBodies(fileDirName, passedTests, failedTests, firstLevelResultsDir,
                                                                      recordPassedMethodBodyFilename);
                                }
                                moveDirToCluster(moveFileToClusterScore, firstLevelResultsDir, allStudentDirs_partition[i], fileName);
                                //recordLevelOneResult(score, studentDir, fileName);
                            }
                        }
                        //MetaProgram.cs didn't build
                        else
                        {
                            //moveSubmissionToUnsuccessful(studentDir, file, null);
                        }
                    }
                }
            }



            /* Cluster level 2 */
            if (approach.ToLower() != "subsumption" && (levelToRun.ToLower() == "second" || levelToRun.ToLower() == "both"))
            {
                string chosenDir = firstLevelResultsDir + "/0";


                Dictionary <string, double> submissionScoreMap = new Dictionary <string, double>();
                string[] allStudentDirs_partition = Directory.GetDirectories(chosenDir);
                //Get all files across allStudentDirs
                var filesArr = Directory.EnumerateDirectories(chosenDir).SelectMany(directory
                                                                                    => Directory.EnumerateFiles(directory, "*.cs"));

                //Conv arr -> List for easier manipulation
                List <string> filesList = new List <string>(filesArr);

                //Insert instructor correct solution at beginning of list (first pass = comparison to ACTUAL soln)
                filesList.Insert(0, instructorSolnDir);
                for (int i = 0; i < filesList.Count - 1; i++)
                {
                    string solutionFile     = filesList[i];
                    string solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);

                    if (solutionFile.EndsWith(".cs"))
                    {
                        //Establish the current file as the ref soln (SubmissionB)
                        if (FileModifier.replaceSolutionMethodBody(className, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
                        {
                            int fileCount = filesArr.Count();
                            //Pair-wise comparison on each file
                            for (int j = i + 1; j < filesList.Count; j++)
                            {
                                string submissionFile = filesList[j];
                                if (submissionFile.EndsWith(".cs"))
                                {
                                    string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);
                                    int    lastIndx           = submissionFile.LastIndexOf('\\');
                                    string temp = submissionFile.Substring(0, lastIndx);
                                    string submissionDirName = temp.Substring(temp.LastIndexOf('\\') + 1);

                                    lastIndx = filesList[i].LastIndexOf('\\');
                                    temp     = filesList[i].Substring(0, lastIndx);
                                    string solutionDirName = temp.Substring(temp.LastIndexOf('\\') + 1);

                                    if (fileCount == 1)
                                    {
                                        recordLevelTwoResult(firstLevelResultsDir, "Cluster: " + solutionDirName
                                                             + "\tReference Submission: " + submissionFileName + "\tMatched");
                                    }
                                    else
                                    {
                                        //Establish submissionA
                                        //Return false if unsuccessful copy (empty method body)
                                        if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, className, hardcodedMetaProjName))
                                        {
                                            //If compiled correctly after having replaced the body
                                            changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
                                            if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
                                            {
                                                //Below line ensures that Pex4Fun uses most updated ver. of asembly file
                                                copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);

                                                RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
                                                //string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
                                                //string solutionFileNameMinusCS = solutionFileName.Substring(0, solutionFileName.Length - 3);
                                                string combinedFileName = submissionDirName + "&" + solutionDirName;
                                                Tuple <HashSet <string[]>, HashSet <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
                                                //Item1 = passedTests, Item2 = failedTests

                                                //string failType = extractedResults.Item3;
                                                //if failType == "assert"... else if failType == "exception"...

                                                //Round score to nearest hundreth
                                                double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, combinedFileName), 2);
                                                recordPassedAndFailedMethodBodies(combinedFileName, extractedResults.Item1, extractedResults.Item2, secondLevelResultsDir, score.ToString());


                                                //TODO:
                                                // Use FIRST pass score to figure out what initial cluster to assign folks to, THEN use 1/0 to figure out whether submissions should be clustered together
                                                //First pass, add all scores to the

                                                //Create mapping of PSE scores assigned during first pass
                                                if (i == 0)
                                                {
                                                    //Add unique passing tests to map
                                                    if (!uniquePassingTestsClusterMap.ContainsKey(score))
                                                    {
                                                        uniquePassingTestsClusterMap.Add(score, new HashSet <string>());
                                                    }
                                                    //Conv string[] -> string, preserving "\n"
                                                    uniquePassingTestsClusterMap[score].Add(string.Join("", extractedResults.Item1.Select(x => string.Join("", x))));

                                                    //Add unique failing tests to map
                                                    if (!uniqueFailingTestsClusterMap.ContainsKey(score))
                                                    {
                                                        uniqueFailingTestsClusterMap.Add(score, new HashSet <string>());
                                                    }
                                                    //Conv string[] -> string
                                                    uniqueFailingTestsClusterMap[score].Add(string.Join("", extractedResults.Item2.Select(x => string.Join("", x))));

                                                    //Add solution name and score to map
                                                    if (!submissionScoreMap.ContainsKey(solutionFileName))
                                                    {
                                                        submissionScoreMap.Add(solutionFileName, score);
                                                    }
                                                    submissionScoreMap[solutionFileName] = score;

                                                    //Add submission name and score to map
                                                    if (!submissionScoreMap.ContainsKey(submissionFileName))
                                                    {
                                                        submissionScoreMap.Add(submissionFileName, score);
                                                    }
                                                    submissionScoreMap[submissionFileName] = score;
                                                }

                                                if (score == 1)
                                                {
                                                    recordLevelTwoResult(firstLevelResultsDir, "Cluster: " + solutionDirName
                                                                         + "\tResultant PSE Score: " + score
                                                                         + "\tReference Submission: " + solutionDirName
                                                                         + "\tCompared Submission: " + submissionDirName
                                                                         + "\tMatched");

                                                    //Add tests pertaining to PSE score assigned during first pass here:
                                                    if (i != 0)
                                                    {
                                                        score = submissionScoreMap[submissionFileName];
                                                    }
                                                    //Add unique passing tests to map
                                                    if (!uniquePassingTestsClusterMap.ContainsKey(score))
                                                    {
                                                        uniquePassingTestsClusterMap.Add(score, new HashSet <string>());
                                                    }
                                                    //Conv string[] -> string, preserving "\n"
                                                    uniquePassingTestsClusterMap[score].Add(string.Join("", extractedResults.Item1.Select(x => string.Join("", x))));

                                                    //Add unique failing tests to map
                                                    if (!uniqueFailingTestsClusterMap.ContainsKey(score))
                                                    {
                                                        uniqueFailingTestsClusterMap.Add(score, new HashSet <string>());
                                                    }
                                                    uniqueFailingTestsClusterMap[score].Add(string.Join("", extractedResults.Item2.Select(x => string.Join("", x))));

                                                    //Cluster those that got matched and remove them from filesList
                                                    filesList.RemoveAt(j);
                                                    j--; //Keep inner ptr in same place after removal
                                                    Console.WriteLine(filesList[j]);
                                                }
                                                else
                                                {
                                                    recordLevelTwoResult(firstLevelResultsDir, "Cluster: " + solutionDirName
                                                                         + "\tResultant PSE Score: " + score
                                                                         + "\tReference Submission: " + solutionDirName
                                                                         + "\tCompared Submission: " + submissionDirName
                                                                         + "\tNot Matched");
                                                }
                                            }
                                        }
                                        //MetaProgram.cs didn't build
                                        else
                                        {
                                            //moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
                                            Console.WriteLine(solutionDirName + " and " + submissionDirName + " failed to produce compiling editedMetaProject");
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }

            //TODO: Keep one test as the reference. Loop over all other tests and call the Except() to figure out the distinct set of tests that the reference has
            //Q from Angello: How to ensure that the unique, minimal test set we're generating per cluster PASSES on every other cluster but fails its own cluster??
            foreach (var reference in uniquePassingTestsClusterMap)
            {
                //Initialize IEnumerable<string> as the first test in dict
                var currTest = uniquePassingTestsClusterMap[reference.Key].Intersect(uniquePassingTestsClusterMap[reference.Key]);
                foreach (var other in uniquePassingTestsClusterMap)
                {
                    if (reference.Key == other.Key)
                    {
                        continue;
                    }
                    //After looping done, currTest is the MIN set of passing tests to differentiate cluster from others
                    currTest = currTest.Intersect(uniquePassingTestsClusterMap[reference.Key]
                                                  .Except(uniquePassingTestsClusterMap[other.Key]));
                }
                recordUniquePassFailClusterTests(currTest, reference.Key, secondLevelResultsDir, "Passed");
            }


            foreach (var reference in uniqueFailingTestsClusterMap)
            {
                var currTest = uniqueFailingTestsClusterMap[reference.Key].Intersect(uniqueFailingTestsClusterMap[reference.Key]);
                foreach (var other in uniqueFailingTestsClusterMap)
                {
                    if (reference.Key == other.Key)
                    {
                        continue;
                    }
                    //After looping done, currTest is the MIN set of failing tests to differentiate cluster from others
                    currTest = currTest.Intersect(uniqueFailingTestsClusterMap[reference.Key]
                                                  .Except(uniqueFailingTestsClusterMap[other.Key]));
                }
                recordUniquePassFailClusterTests(currTest, reference.Key, secondLevelResultsDir, "Failed");
            }



            /* Level 2: Subsumption */
            if (approach.ToLower() == "subsumption")
            {
                string chosenDir = secondLevelResultsDir;
                if (!Directory.Exists(secondLevelResultsDir))
                {
                    chosenDir = secondLevelResultsDirAlt; //Choose alternate secondLevelDir if orig. doesn't exist!
                }
                foreach (var pseScoreFolder in Directory.GetDirectories(chosenDir))
                {
                    Console.WriteLine("Evaluating PSE FOLDER: " + pseScoreFolder.Substring(pseScoreFolder.LastIndexOf("\\") + 1));
                    string[] allChildClusters = Directory.GetDirectories(pseScoreFolder);
                    for (int i = 0; i < allChildClusters.Length - 1; i++)
                    {
                        string currChild = allChildClusters[i];
                        //Skip the PSE = 1 and PSE = NaN folders
                        if (currChild.Substring(currChild.LastIndexOf("\\") + 1) == "1" ||
                            currChild.Contains("NaN"))
                        {
                            continue;
                        }
                        string[] solutionFilesList = Directory.GetFiles(currChild);      //Get all files within currChild
                        Random   r                = new Random();
                        int      randIndx         = r.Next(0, solutionFilesList.Length); //[0, filesList.Length)
                        string   solutionFile     = solutionFilesList[randIndx];         //Ref Soln = rand file inside child
                        string   solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);

                        if (solutionFile.EndsWith(".cs"))
                        {
                            //Establish the current file as the ref soln
                            if (FileModifier.replaceSolutionMethodBody(className, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
                            {
                                string nextChild = allChildClusters[i + 1]; //Get next child
                                if (nextChild.Substring(nextChild.LastIndexOf("\\") + 1) == "1" ||
                                    nextChild.Contains("NaN"))
                                {
                                    continue;
                                }
                                string[] submissionFilesList = Directory.GetFiles(nextChild); //Get all files within nextChild
                                randIndx = r.Next(0, submissionFilesList.Length);             //[0, filesList.Length)
                                string submissionFile     = submissionFilesList[randIndx];    //Ref Soln = rand file inside child
                                string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);


                                if (submissionFile.EndsWith(".cs"))
                                {
                                    Console.WriteLine("Both " + solutionFileName + " and " + submissionFileName + " end in .cs");

                                    //Begin replacing method body of submission
                                    //Return false if unsuccessful copy (empty method body)
                                    if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, className, hardcodedMetaProjName))
                                    {
                                        //If compiled correctly after having replaced the body
                                        changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
                                        if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
                                        {
                                            //Below line ensures that Pex4Fun uses most updated ver. of asembly file
                                            copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);

                                            RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
                                            string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
                                            string solutionFileNameMinusCS   = solutionFileName.Substring(0, solutionFileName.Length - 3);
                                            string combinedFileName          = solutionFileNameMinusCS + "&" + submissionFileNameMinusCS;
                                            Tuple <HashSet <string[]>, HashSet <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
                                            string failType = extractedResults.Item3;

                                            string message = "";
                                            if (failType == "assertion")
                                            {
                                                Console.WriteLine("s2 >= s1");
                                                message = "Reference solution subsumes student submission.";
                                            }
                                            else if (failType == "exception")
                                            {
                                                Console.WriteLine("s1 >= s2");
                                                message = "Student submission subsumes reference solution.";
                                            }
                                            else if (failType == "")
                                            {
                                                message = "No subsumption.";
                                            }
                                            else
                                            {
                                                message = "Neither...something likely went wrong.";
                                            }

                                            //Record to file in subsumption folder
                                            recordSubsumptionResult(subsumptionResultsDir, combinedFileName, message);
                                        }
                                        //MetaProgram.cs didn't build
                                        else
                                        {
                                            //moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
                                            Console.WriteLine(solutionFileName + " and " + submissionFileName + " failed to produce compiling editedMetaProject");
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } //End Main()
Beispiel #3
0
        //GENERAL NOTES (12/19/18):
        // 1. MUST run Program.cs W/O debugging (Toolbar -> Debug -> Start w/o Debugging)
        // 2. CANNOT have the ...\editedMetaProject\bin\Debug\reports folder open

        public static void Main(string[] args)
        {
            string problemName = args[0];
            string approach    = args[1];
            string filterType  = args[2];
            string levelToRun  = args[3];

            //Set that Pex4Fun.exe runs from as current directory
            string currDir = System.IO.Directory.GetParent(Environment.CurrentDirectory).ToString();
            //Move 4 levels up to set the top directory
            //Go from C:...codesimilarity\Pex4Fun\Sp18_Q10_20_sorted\behavioralsimilarity\Pex4Fun\bin\Debug\ -> C:...\codesimilarity\
            string topDir = Path.GetFullPath(Path.Combine(currDir, @"..\..\..\..\..\..\"));
            string editedMetaProjectDir     = topDir + @"Approach\" + approach + @"\" + problemName + @"\editedMetaProject";
            string allStudentsDir           = topDir + @"Students\" + problemName;
            string firstLevelResultsDir     = topDir + @"Results\FirstLevel\" + problemName + @"\" + filterType;
            string secondLevelResultsDir    = topDir + @"Results\SecondLevel\Approach\CompleteEquivalence\" + problemName + @"\" + filterType;
            string secondLevelResultsDirAlt = topDir + @"Results\SecondLevel\Approach\NewEquivalence\" + problemName + @"\" + filterType;
            string subsumptionResultsDir    = topDir + @"Results\SecondLevel\Approach\Subsumption\" + problemName;
            string assemblyName             = "meta_project1525456207";
            string hardcodedMetaProjName    = "meta_project1525456207";
            string methodName = "sort";
            Random randNum    = new Random();
            //Dictionary key maps to List -> List -> string arrays...lets me have mult. vals. map to single key
            Dictionary <double, List <List <string[]> > > scoreToTestBodyMap = new Dictionary <double, List <List <string[]> > >();

            /* Cluster level one */
            if (levelToRun.ToLower() == "first" || levelToRun.ToLower() == "both")
            {
                //Choose each folder that ends in "@illinois.edu"
                foreach (var studentDir in Directory.GetDirectories(allStudentsDir))
                {
                    if (studentDir.Contains("@illinois.edu"))
                    {
                        foreach (var file in Directory.GetFiles(studentDir))
                        {
                            if (file.EndsWith(".cs"))
                            {
                                //Below false IFF empty addToEnd method body
                                if (FileModifier.CopyBodyToMetaProgram(file, editedMetaProjectDir, methodName, hardcodedMetaProjName))
                                {
                                    //Account for default / non-default Pex emission filter
                                    changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
                                    if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
                                    {
                                        //Below line ensures that Pex4Fun uses most updated ver. of asembly file
                                        copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);

                                        string fileName = file.Substring(file.LastIndexOf("\\") + 1);
                                        //12/03/18, by Jonathan: Method to run pex on editedMetaProject's MetaProgram.cs
                                        RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
                                        string fileNameMinusCS = fileName.Substring(0, fileName.Length - 3);
                                        Tuple <List <string[]>, List <string[]>, string> passedAndFailedTests = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, fileNameMinusCS, "MetaProgram", "Check", assemblyName);
                                        List <string[]> passedTests = passedAndFailedTests.Item1;
                                        List <string[]> failedTests = passedAndFailedTests.Item2;
                                        //Round score to nearest hundreth
                                        double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, fileNameMinusCS), 2);

                                        //string moveFileToClusterFileName = fileName;
                                        string moveFileToClusterScore         = score.ToString();
                                        string recordPassedMethodBodyFilename = score.ToString();
                                        bool   areTestsIdentical = false;

                                        //If there were 1+ passing tests...
                                        if (passedTests.Count != 0)
                                        {
                                            List <List <string[]> > existantTestBodiesList = new List <List <string[]> >();
                                            //FIRST check if key already exists in Dictionary before adding...
                                            if (scoreToTestBodyMap.TryGetValue(score, out existantTestBodiesList))
                                            {
                                                //Check if both lists passed on SAME tests, regardless of order / dupe entries
                                                var list1Lookup = new HashSet <string[]>(passedTests);
                                                foreach (var methodBodies in existantTestBodiesList)
                                                {
                                                    var list2Lookup = new HashSet <string[]>(methodBodies);
                                                    if (list1Lookup.SetEquals(list2Lookup))
                                                    {
                                                        //If they've got same values, then they truly belong in same parent cluster
                                                        recordPassedMethodBodyFilename = score.ToString();
                                                        moveFileToClusterScore         = score.ToString();
                                                        areTestsIdentical = true;
                                                        break;
                                                    }
                                                    else
                                                    {
                                                        //Else they belong in same parent cluster, but DIFF child clusters
                                                        string randNumStr = randNum.Next(0, 100).ToString();
                                                        moveFileToClusterScore         = score.ToString() + "_" + randNumStr;
                                                        recordPassedMethodBodyFilename = score.ToString() + @"\" + randNumStr;
                                                    }
                                                }
                                            }
                                            //Else if the key didn't already exist in Dictionary...
                                            //...Store the list in memory IFF tests are NOT identical...
                                            if (!areTestsIdentical)
                                            {
                                                scoreToTestBodyMap.Add(score, new List <List <string[]> >());
                                                scoreToTestBodyMap[score].Add(passedTests);
                                            }
                                            //...Store list in file
                                            recordPassedAndFailedMethodBodies(passedTests, failedTests, firstLevelResultsDir,
                                                                              recordPassedMethodBodyFilename);
                                        }
                                        moveFileToCluster(moveFileToClusterScore, firstLevelResultsDir, file, fileName);
                                        //recordLevelOneResult(score, studentDir, fileName);
                                    }
                                }
                                //MetaProgram.cs didn't build
                                else
                                {
                                    //moveSubmissionToUnsuccessful(studentDir, file, null);
                                }
                            }
                        }
                    }
                }
            }



            /* Cluster level 2 */
            if (approach.ToLower() != "subsumption" && (levelToRun.ToLower() == "second" || levelToRun.ToLower() == "both"))
            {
                //Skip the PSE = 1 and PSE = NaN folders
                foreach (var pseScoreFolder in Directory.GetDirectories(firstLevelResultsDir))
                {
                    if (pseScoreFolder.Substring(pseScoreFolder.LastIndexOf("\\") + 1) == "1" ||
                        pseScoreFolder.Contains("NaN") || pseScoreFolder.Contains("Passed&FailedTests"))
                    {
                        continue;
                    }
                    int      solutionFileIndex = 0;
                    string[] filesList         = Directory.GetFiles(pseScoreFolder);
                    foreach (var solutionFile in filesList)
                    {
                        string solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);

                        if (solutionFile.EndsWith(".cs"))
                        {
                            //Establish the current file as the ref soln
                            if (FileModifier.replaceSolutionMethodBody(methodName, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
                            {
                                int fileCount           = Directory.GetFiles(pseScoreFolder).Length;
                                int submissionFileIndex = 0;
                                //Pair-wise comparison on each file
                                foreach (var submissionFile in filesList)
                                {
                                    if (submissionFile.EndsWith(".cs"))
                                    {
                                        string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);
                                        string pseScoreFolderName = pseScoreFolder.Substring(pseScoreFolder.LastIndexOf('\\') + 1);
                                        if (fileCount == 1)
                                        {
                                            recordLevelTwoResult(pseScoreFolder, "PSE Cluster: " + pseScoreFolderName
                                                                 + "\tReference Submission: " + submissionFileName + "\tMatched");
                                        }
                                        else if (submissionFileIndex > solutionFileIndex)
                                        {
                                            //Begin replacing method body of submission
                                            //Return false if unsuccessful copy (empty method body)
                                            if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, methodName, hardcodedMetaProjName))
                                            {
                                                //If compiled correctly after having replaced the body
                                                changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
                                                if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
                                                {
                                                    //Below line ensures that Pex4Fun uses most updated ver. of asembly file
                                                    copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);

                                                    RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
                                                    string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
                                                    string solutionFileNameMinusCS   = solutionFileName.Substring(0, solutionFileName.Length - 3);
                                                    string combinedFileName          = solutionFileNameMinusCS + "&" + submissionFileNameMinusCS;
                                                    Tuple <List <string[]>, List <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
                                                    //string failType = extractedResults.Item3;
                                                    //if failType == "assert"... else if failType == "exception"...

                                                    //Round score to nearest hundreth
                                                    double score = Math.Round(scoreFromPexTestsXML(editedMetaProjectDir, combinedFileName), 2);
                                                    if (score == 1)
                                                    {
                                                        recordLevelTwoResult(pseScoreFolder, "PSE Cluster: " + pseScoreFolderName
                                                                             + "\tResultant PSE Score: " + score
                                                                             + "\tReference Submission: " + solutionFileName
                                                                             + "\tCompared Submission: " + submissionFileName
                                                                             + "\tMatched");
                                                        //Remove submissionFile from filesList if score == 1 (more efficient pairwise algorithm)
                                                        filesList = filesList.Where(w => w != filesList[submissionFileIndex]).ToArray();
                                                        submissionFileIndex--; //Decrem then increm submissionFileIndex to keep value the same
                                                    }
                                                    else
                                                    {
                                                        recordLevelTwoResult(pseScoreFolder, "PSE Cluster: " + pseScoreFolderName
                                                                             + "\tResultant PSE Score: " + score
                                                                             + "\tReference Submission: " + solutionFileName
                                                                             + "\tCompared Submission: " + submissionFileName
                                                                             + "\tNot Matched");
                                                    }
                                                }
                                            }
                                            //MetaProgram.cs didn't build
                                            else
                                            {
                                                //moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
                                                Console.WriteLine(solutionFileName + " and " + submissionFileName + " failed to produce compiling editedMetaProject");
                                            }
                                        }
                                    }
                                    submissionFileIndex++;
                                }
                            }
                        }
                        solutionFileIndex++;
                    }
                }
            }

            /* Level 2: Subsumption */
            if (approach.ToLower() == "subsumption")
            {
                string chosenDir = secondLevelResultsDir;
                if (!Directory.Exists(secondLevelResultsDir))
                {
                    chosenDir = secondLevelResultsDirAlt; //Choose alternate secondLevelDir if orig. doesn't exist!
                }
                string[] allParentClusters = Directory.GetDirectories(chosenDir);
                for (int j = 0; j < allParentClusters.Length; j++)
                {
                    string pseScoreFolder = allParentClusters[j];
                    Console.WriteLine("Evaluating PSE FOLDER: " + pseScoreFolder.Substring(pseScoreFolder.LastIndexOf("\\") + 1));
                    string[] allChildClusters = Directory.GetDirectories(pseScoreFolder);
                    for (int i = 0; i < allChildClusters.Length; i++)
                    {
                        string currChild = allChildClusters[i];
                        Console.WriteLine("CURRCHILD = " + currChild);
                        //Skip the PSE = 1 and PSE = NaN folders
                        if (currChild.Substring(currChild.LastIndexOf("\\") + 1) == "1" ||
                            currChild.Contains("NaN"))
                        {
                            continue;
                        }
                        string[] solutionFilesList = Directory.GetFiles(currChild);      //Get all files within currChild
                        Random   r                = new Random();
                        int      randIndx         = r.Next(0, solutionFilesList.Length); //[0, filesList.Length)
                        string   solutionFile     = solutionFilesList[randIndx];         //Ref Soln = rand file inside child
                        string   solutionFileName = solutionFile.Substring(solutionFile.LastIndexOf("\\") + 1);

                        if (solutionFile.EndsWith(".cs"))
                        {
                            //Establish the current file as the ref soln
                            if (FileModifier.replaceSolutionMethodBody(methodName, solutionFile, editedMetaProjectDir + @"\MetaProgram.cs"))
                            {
                                //TO EDIT (04/05/2019): Need to select ONE other file from each child cluster inside...
                                //...DIFF parent cluster!!
                                for (int k = j + 1; k < allParentClusters.Length; k++)
                                {
                                    string   nextPseScoreFolder   = allParentClusters[k];
                                    string[] allNextChildClusters = Directory.GetDirectories(nextPseScoreFolder);
                                    for (int x = 0; x < allNextChildClusters.Length; x++)
                                    {
                                        string nextChild = allChildClusters[x]; //Get next child
                                        if (nextChild.Substring(nextChild.LastIndexOf("\\") + 1) == "1" ||
                                            nextChild.Contains("NaN"))
                                        {
                                            continue;
                                        }
                                        string[] submissionFilesList = Directory.GetFiles(nextChild); //Get all files within nextChild
                                        randIndx = r.Next(0, submissionFilesList.Length);             //[0, filesList.Length)
                                        string submissionFile     = submissionFilesList[randIndx];    //Ref Soln = rand file inside child
                                        string submissionFileName = submissionFile.Substring(submissionFile.LastIndexOf("\\") + 1);


                                        if (submissionFile.EndsWith(".cs"))
                                        {
                                            Console.WriteLine("Both " + solutionFileName + " and " + submissionFileName + " end in .cs");

                                            //Begin replacing method body of submission
                                            //Return false if unsuccessful copy (empty method body)
                                            if (FileModifier.CopyBodyToMetaProgram(submissionFile, editedMetaProjectDir, methodName, hardcodedMetaProjName))
                                            {
                                                //If compiled correctly after having replaced the body
                                                changeFilterType(filterType, editedMetaProjectDir + @"\MetaProgram.cs");
                                                if (BuildDirectory.BuildSingleProject(editedMetaProjectDir, true))
                                                {
                                                    //Below line ensures that Pex4Fun uses most updated ver. of asembly file
                                                    copyOverAssemblyFile(editedMetaProjectDir, currDir, assemblyName);

                                                    RunPex.RunPexOnEditedMetaProject(editedMetaProjectDir, assemblyName);
                                                    string submissionFileNameMinusCS = submissionFileName.Substring(0, submissionFileName.Length - 3);
                                                    string solutionFileNameMinusCS   = solutionFileName.Substring(0, solutionFileName.Length - 3);
                                                    string combinedFileName          = solutionFileNameMinusCS + "&" + submissionFileNameMinusCS;
                                                    Tuple <List <string[]>, List <string[]>, string> extractedResults = ExtractPexResults.ExtractTestsForEditedMetaProject(editedMetaProjectDir, combinedFileName, "MetaProgram", "Check", assemblyName);
                                                    string failType = extractedResults.Item3;

                                                    string message = "";
                                                    if (failType == "assertion")
                                                    {
                                                        Console.WriteLine("s2 >= s1");
                                                        message = "Reference solution subsumes student submission.";
                                                    }
                                                    else if (failType == "exception")
                                                    {
                                                        Console.WriteLine("s1 >= s2");
                                                        message = "Student submission subsumes reference solution.";
                                                    }
                                                    else if (failType == "")
                                                    {
                                                        message = "No subsumption.";
                                                    }
                                                    else
                                                    {
                                                        message = "Neither...something likely went wrong.";
                                                    }

                                                    //Record to file in subsumption folder
                                                    recordSubsumptionResult(subsumptionResultsDir, combinedFileName, message);
                                                }
                                                //MetaProgram.cs didn't build
                                                else
                                                {
                                                    //moveSubmissionToUnsuccessful(pseScoreFolder, solutionFileName, submissionFileName);
                                                    Console.WriteLine(solutionFileName + " and " + submissionFileName + " failed to produce compiling editedMetaProject");
                                                }
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        } //End Main()