public void appendProjectPathsToGHTorrentCSV()
{
string ghMainProjectsFile = string.Format($"{clonedProjectsFolder}\\ghprojectsMain.csv");
string[] lines = File.ReadAllLines(ghprojectsFile);
lines = lines.Distinct().ToArray();
string header = string.Format($"{lines[0]};projectPath;parentPath;ecosystem;projectName");
MiningUtility.createFileHeader(ghMainProjectsFile, header);
StreamWriter writer = File.AppendText(ghMainProjectsFile);
for (int i = 0; i < lines.Length; i++)
{
string line = lines[i];
int count = i;
try
{
GHProject p = getrojectFolder(line);
if (p != null)
{
writer.WriteLine($"{line};{p.ProjectFolder};{p.ParentPath};{p.ProjectName};{p.ProjectName}");
}
//Task cloneTask = Task.Factory.StartNew(() => doClone(bag, line, writer, count, legnth));
//tasks.Add(cloneTask);
Console.WriteLine($"Ecosystem number:{i}/{lines.Length}");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message + "\n" + ex.StackTrace);
}
}
writer.Close();
}
public void generateFinalMainlineGHProjectsCSV()
{
//generating csv with final mainline repos
string finalProjectsFile = string.Format($"{clonedProjectsFolder}\\ghprojectsFinalMainline.csv");
string header = "id;projectname;createdat;testcases";
MiningUtility.createFileHeader(finalProjectsFile, header);
StreamWriter writer = File.AppendText(finalProjectsFile);
DataController dc = new DataController();
SqlDataReader reader = dc.GetMainlineReposWithMoreThan20TestCases();
Console.WriteLine("Generating CSv with mainline repos that have more than 20 test cases: " + finalProjectsFile);
while (reader.Read())
{
writer.WriteLine($"{reader["ProjectID"]};{reader["ProjectName"]};{reader["CreatedAt"]};{reader["testcases"]}");
}
reader.Close();
writer.Close();
}
public void detectTestCases()
{
DataController dc = new DataController();
var projects = MiningUtility.GetProjectGroups();
int projectCount = 0, projectsNumber = projects.Count();
string testCaseFileHeader = "project;parent;projectIsFork;testPackage;testClassName;testMethodName;testMethodParameters;testSourceFilePath;testStartingLine;testEndingLine;testStartingColumn;testEndingColumn;uutPackage;uutClassName;uutMethodName;uutMethodParameters;uutSourceFilePath;uutStartingLine;uutEndingLine;uutStartingColumn;uutEndingColumn";
MiningUtility.createFileHeader(testCasesFile, testCaseFileHeader);
string classesHeader = "project;parent;projectIsFork;package;className;methodName;methodParameters;SourceFilePath;StartingLine;EndingLine;StartingColumn;EndingColumn";
MiningUtility.createFileHeader(classesAndMethodsFile, classesHeader);
//extract clones per ecosystem
foreach (ProjectGroup project in projects)
{
projectCount++;
try
{
string projectFolder = project.ProjectFolder;// "C:\\grouped\\3_android_packages_apps_Contacts\\LineageOS\\android_packages_apps_Contacts";
TestCaseDetector testCaseDetector = new TestCaseDetector(projectFolder, srcMLLocation);
testCaseDetector.AnalyzeTestCases();
HashSet <TestCaseID> allTestCases = testCaseDetector.AllTestCases;
HashSet <TestCaseSummary> projectTestSummary = null; // new HashSet<TestCaseSummary>();
//projct parent
string parent = project.Ecosystem;
string projectName = project.Project;
bool isFork = projectName != parent;
Utilities.analyzeProjectTestCases(projectFolder, allTestCases, dc, srcMLLocation, testCasesFile, classesAndMethodsFile, projectName, parent, isFork);
Console.WriteLine($"Ecosystem: {project.Ecosystem}; Project:{projectCount}/{projectsNumber}");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message + "\n" + ex.StackTrace);
}
}
}
public void cloneRepos()
{
CultureInfo culture = CultureInfo.CreateSpecificCulture("en-US");
Thread.CurrentThread.CurrentCulture = culture;
Thread.CurrentThread.CurrentUICulture = culture;
setForkedRepos();
//setPullRequests();
List <string> projects = new List <string>();
List <string> parents = new List <string>();
foreach (ForkedRepo repo in forkedRepos)
{
if (!projects.Contains(repo.ParentRepo))
{
projects.Add(repo.ParentRepo);
parents.Add(repo.ParentRepo);
}
projects.Add(repo.ForkName);
}
//now clone repos
string folderName = ConfigurationManager.AppSettings["clonedProjects"];
string commands = ConfigurationManager.AppSettings["commands"];
//string[] doneProjects = new string[] { "LineageOS/android_packages_apps_Contacts", "LightningFastRom/android_packages_apps_Contacts", "sdhz152/android_packages_apps_Contacts", "Michenux/YourAppIdea", "wendersonferreira/YourAppIdea", "friederbluemle/YourAppIdea", "PatilShreyas/MaterialDialog-Android", "tetrapi/MaterialDialog-Android", "xuexiangjys/TemplateAppProject", "myie9/TemplateAppProject", "badoo/Chateau", "wiyarmir/Chateau", "segler-alex/RadioDroid", "kar10s/RadioDroid", "morckx/RadioDroid", "werman/RadioDroid", "robotmedia/AndroidBillingLibrary", "masconsult/AndroidBillingLibrary", "guetux/AndroidBillingLibrary", "hpique/AndroidBillingLibrary", "serso/android-billing", "azhon/AppUpdate", "hongyantao/AppUpdate", "burgessjp/ThemeSkinning", "humanheima/ThemeSkinning", "msdx/ThemeSkinning", "athkalia/Just-Another-Android-App", "MaTriXy/Just-Another-Android-App", "segmentio/analytics-android", "friederbluemle/analytics-android", "super-collider/analytics-android", "TorreyKahuna/analytics-android", "rayleeriver/analytics-android", "amplitude/analytics-android", "google/bundletool" };
string[] finalProjects = MiningUtility.GetFinalProjects();
try
{
List <ForkedRepo> repos = MiningUtility.GetForkedRepos();
var repoParents = from tcp in repos
select tcp.ParentRepo;
repoParents = repoParents.Distinct();
int parentCount = 0;
int parentsNumber = parents.Count();
foreach (string parent in repoParents)
{
parentCount++;
if (parent == null || parent == "null")
{
continue;
}
if (!finalProjects.Contains(parent))
{
continue;
}
//get ecosystem projects
var ecosystemProjects = from tcp in repos.Where(p => p.ForkName == parent || p.ParentRepo == parent)
select tcp.ForkName;
List <string> ecoProjects = ecosystemProjects.ToList();
ecoProjects.Add(parent);
ecoProjects = ecoProjects.Distinct().ToList();
//create parent folder
string parentFolder = string.Format($"{parentCount}_{parent.Split('/')[1]}");//if parent is muke/android, and count is 12, the folder will be 12_android
string parentPath = string.Format($"E:\\grouped\\{parentFolder}");
//now copy every ecosystem project to parent directory
int projectCount = 0;
int projectsNumber = ecoProjects.Count();
foreach (string project in ecoProjects)
{
projectCount++;
if (!finalProjects.Contains(project))
{
continue;
}
string projectFolder = string.Format($"{parentPath}\\{project.Replace("/", "\\")}");
string cloneURL = string.Format($"https://github.com/{project}.git");
Repository.Clone(cloneURL, projectFolder);
Console.WriteLine($"Ecosystem: {parent}, number:{parentCount}/{parentsNumber - 1};Pair:{projectCount}/{projectsNumber}");
}
}
//foreach (string project in projects)
//{
// if (!finalProjects.Contains(project))
// {
// continue;
// }
// //string projectSHortName = project.Split('/')[1];
// string projectFolder = string.Format($"{folderName}\\{project.Replace("/", "\\")}");
// string cloneURL = string.Format($"https://github.com/{project}.git");
// //projectURLs.TryGetValue(project, out cloneURL);
// //DirectoryInfo directoryInfo = new DirectoryInfo(projectFolder);
// //if (directoryInfo.Exists)
// //{
// // directoryInfo.Delete(true);
// //}
// Console.WriteLine($"cloning {project}");
// try
// {
// string repoPath = Repository.Clone(cloneURL, projectFolder);
// }
// catch (Exception ex)
// {
// Console.WriteLine(ex.Message + "\n" + ex.StackTrace);
// }
//}
}
catch (Exception ex)
{
Console.WriteLine(ex.StackTrace);
}
finally
{
}
}
public void setPullRequests()
{
pullRequests = MiningUtility.getPullRequests(File.ReadAllLines(pullRequestsFile).ToList());
}
public void setForkedRepos()
{
forkedRepos = MiningUtility.GetForkedRepos();
}